def run(args, command_name="mmtbx.csv_to_mtz"):
if (len(args) == 0): args = ["--help"]
try:
command_line = (iotbx_option_parser(
usage="%s [reflection_csv_file] [options]" % command_name,
description='Example: %s 1m5u-sf.csv --use_model=1m5u.pdb' %
command_name
).enable_symmetry_comprehensive().option(
None,
"--output_file",
action="store",
default=False,
type="string",
help="Output mtz file name."
).option(
None,
"--use_model",
action="store",
default=False,
type="string",
help=
"Use PDB model to make better guess about reflection data type.")
).process(args=args)
except Exception, e:
if (str(e) != "0"): print str(e)
sys.exit(0)
def process_zeolite_atlas(args):
command_line = (iotbx_option_parser()
.option(None, "--tag",
action="store",
type="string")
.option(None, "--verbose",
action="store_true")
).process(args=args)
atlas_file = libtbx.env.find_in_repositories(
relative_path="phenix_regression/misc/strudat_zeolite_atlas",
test=os.path.isfile)
if (atlas_file is None):
print "Skipping process_zeolite_atlas(): input file not available"
return
if (command_line.options.verbose):
out = sys.stdout
else:
out = StringIO()
all_entries = strudat.read_all_entries(open(atlas_file))
for i,entry in enumerate(all_entries.entries):
structure = entry.as_xray_structure()
if (command_line.options.tag is not None):
if (command_line.options.tag != entry.tag):
continue
print >> out, "strudat tag:", entry.tag
exercise(structure=structure, out=out)
exercise_shake_sites_in_place(structure=structure)
def exercise(args, distance_cutoff=3.5, max_shell=5):
command_line = (iotbx_option_parser()
.option(None, "--tag",
action="store",
type="string")
.option(None, "--full",
action="store_true")
.option(None, "--verbose",
action="store_true")
).process(args=args)
co = command_line.options
atlas_file = libtbx.env.find_in_repositories(
relative_path="phenix_regression/misc/strudat_zeolite_atlas",
test=os.path.isfile)
if (atlas_file is None):
print "Skipping exercise(): input file not available"
return
all_entries = strudat.read_all_entries(open(atlas_file))
for i,entry in enumerate(all_entries.entries):
structure = entry.as_xray_structure()
if (co.tag is not None):
if (co.tag != entry.tag):
continue
elif (not (co.full or i % 20 == 0)):
continue
if (co.verbose):
print "strudat tag:", entry.tag
exercise_simple(
structure, distance_cutoff, max_shell, co.verbose)
exercise_shell_asu_tables(structure, co.verbose)
def process_zeolite_atlas(args):
command_line = (iotbx_option_parser().option(
None, "--tag", action="store",
type="string").option(None, "--verbose",
action="store_true")).process(args=args)
atlas_file = libtbx.env.find_in_repositories(
relative_path="phenix_regression/misc/strudat_zeolite_atlas",
test=os.path.isfile)
if (atlas_file is None):
print("Skipping process_zeolite_atlas(): input file not available")
return
if (command_line.options.verbose):
out = sys.stdout
else:
out = StringIO()
with open(atlas_file) as f:
all_entries = strudat.read_all_entries(f)
for i, entry in enumerate(all_entries.entries):
structure = entry.as_xray_structure()
if (command_line.options.tag is not None):
if (command_line.options.tag != entry.tag):
continue
print("strudat tag:", entry.tag, file=out)
exercise(structure=structure, out=out)
exercise_shake_sites_in_place(structure=structure)
def run(args, command_name = "phenix.cif_as_mtz", out=sys.stdout,
return_as_miller_arrays=False):
if (len(args) == 0): args = ["--help"]
try:
command_line = (iotbx_option_parser(
usage="%s [reflection_cif_file] [options]" % command_name,
description='Example: %s r1o9ksf.ent --symmetry=pdb1o9k.ent'%command_name)
.enable_symmetry_comprehensive()
.option(None, "--output_file_name",
action="store",
default=False,
type="string",
help="Output mtz file name.")
.option(None, "--wavelength_id",
action="store",
default=None,
type="int",
help="Extract data set with given wavelength_id.")
.option(None, "--crystal_id",
action="store",
default=None,
type="int",
help="Extract data set with given crystal_id.")
.option(None, "--output_r_free_label",
action="store",
default="R-free-flags",
type="string",
help="MTZ column label to use for R-free flags (default: R-free-flags)")
.option(None, "--merge",
action="store_true",
help="Merge non-unique data where present.")
.option(None, "--incompatible_flags_to_work_set",
action="store_true",
help="When merging place reflections with incompatible flags into the "
"working set.")
.option(None, "--remove_systematic_absences",
action="store_true",
help="Remove systematic absent reflections.")
.option(None, "--map_to_asu",
action="store_true",
help="Map to asymmetric unit.")
.option("--show_details_if_error",
action="store_true",
help="Show data details for some errors.")
.option("--show_log",
action="store_true",
help="Show some output.")
.option("--ignore_bad_sigmas",
action="store_true",
help="Set sigmas to None instead of raising an error when bad sigmas "
"are present.")
.option("--extend_flags",
action="store_true",
help="Extend R-free flags to cover all reflections if necessary.")
).process(args=args)
except Exception, e:
if(str(e) != "0"): print str(e)
sys.exit(0)
def run(args, command_name="phenix.tls"):
if (len(args) == 0): args = ["--help"]
usage_fmt = "%s pdb_file [parameters: file or command line string]"
des_fmt = "Example: %s model.pdb fit_tls_to.selection='%s' fit_tls_to.selection='%s'"
command_line = (iotbx_option_parser(
usage=usage_fmt % command_name, description=banner).option(
"--show_defaults",
action="store_true",
help="Do not output to the screen (except errors).").option(
"--silent",
action="store_true",
help="Suppress output to the screen.")).process(args=args)
#
log = sys.stdout
if (not command_line.options.silent):
utils.print_header("TLS tools", out=log)
if (command_line.options.show_defaults):
master_params.show(out=log)
print >> log
return
if (not command_line.options.silent):
print >> log, banner
#
processed_args = utils.process_command_line_args(
args=command_line.args, master_params=master_params, log=log)
reflection_files = processed_args.reflection_files
if (processed_args.crystal_symmetry is None):
raise Sorry("No crystal symmetry found.")
if (len(processed_args.pdb_file_names) == 0):
raise Sorry("No PDB file found.")
params = processed_args.params
if (not command_line.options.silent):
utils.print_header("Input parameters", out=log)
params.show(out=log)
params = params.extract()
#
if (processed_args.crystal_symmetry.unit_cell() is None
or processed_args.crystal_symmetry.space_group() is None):
raise Sorry("No CRYST1 record found.")
pdb_combined = iotbx.pdb.combine_unique_pdb_files(
file_names=processed_args.pdb_file_names)
pdb_combined.report_non_unique(out=log)
if (len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
raw_records = pdb_combined.raw_records
try:
pdb_inp = iotbx.pdb.input(source_info=None,
lines=flex.std_string(raw_records))
except ValueError, e:
raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e))
def run(args, command_name = "phenix.nm"):
if(len(args) == 0): args = ["--help"]
usage_fmt = "%s pdb_file [parameters: file or command line string]"
des_fmt = "Example: %s model.pdb fit_tls_to.selection='%s' fit_tls_to.selection='%s'"
command_line = (iotbx_option_parser(
usage = usage_fmt % command_name,
description = banner)
.option("--show_defaults",
action="store_true",
help="Do not output to the screen (except errors).")
.option("--silent",
action="store_true",
help="Suppress output to the screen.")
.option("--use_elbow",
action="store_false",
help="Use eLBOW if there are unknown ligands in PDB file (available in PHENIX only).")
).process(args=args)
def run(args, command_name = "mmtbx.csv_to_mtz"):
if (len(args) == 0): args = ["--help"]
try:
command_line = (iotbx_option_parser(
usage="%s [reflection_csv_file] [options]" % command_name,
description='Example: %s 1m5u-sf.csv --use_model=1m5u.pdb'%command_name)
.enable_symmetry_comprehensive()
.option(None, "--output_file",
action="store",
default=False,
type="string",
help="Output mtz file name.")
.option(None, "--use_model",
action="store",
default=False,
type="string",
help="Use PDB model to make better guess about reflection data type.")
).process(args=args)
except Exception, e:
if(str(e) != "0"): print str(e)
sys.exit(0)
def exercise(args, distance_cutoff=3.5, max_shell=5):
command_line = (iotbx_option_parser().option(
None, "--tag", action="store",
type="string").option(None, "--full", action="store_true").option(
None, "--verbose", action="store_true")).process(args=args)
co = command_line.options
atlas_file = libtbx.env.find_in_repositories(
relative_path="phenix_regression/misc/strudat_zeolite_atlas",
test=os.path.isfile)
if (atlas_file is None):
print "Skipping exercise(): input file not available"
return
all_entries = strudat.read_all_entries(open(atlas_file))
for i, entry in enumerate(all_entries.entries):
structure = entry.as_xray_structure()
if (co.tag is not None):
if (co.tag != entry.tag):
continue
elif (not (co.full or i % 20 == 0)):
continue
if (co.verbose):
print "strudat tag:", entry.tag
exercise_simple(structure, distance_cutoff, max_shell, co.verbose)
exercise_shell_asu_tables(structure, co.verbose)
def run(args,
command_name="phenix.cif_as_mtz",
out=sys.stdout,
return_as_miller_arrays=False):
if (len(args) == 0): args = ["--help"]
try:
command_line = (iotbx_option_parser(
usage="%s [reflection_cif_file] [options]" % command_name,
description='Example: %s r1o9ksf.ent --symmetry=pdb1o9k.ent' %
command_name
).enable_symmetry_comprehensive().option(
None,
"--output_file_name",
action="store",
default=False,
type="string",
help="Output mtz file name."
).option(
None,
"--wavelength_id",
action="store",
default=None,
type="int",
help="Extract data set with given wavelength_id."
).option(
None,
"--crystal_id",
action="store",
default=None,
type="int",
help="Extract data set with given crystal_id."
).option(
None,
"--output_r_free_label",
action="store",
default="R-free-flags",
type="string",
help=
"MTZ column label to use for R-free flags (default: R-free-flags)"
).option(
None,
"--merge",
action="store_true",
help="Merge non-unique data where present."
).option(
None,
"--incompatible_flags_to_work_set",
action="store_true",
help=
"When merging place reflections with incompatible flags into the "
"working set."
).option(
None,
"--remove_systematic_absences",
action="store_true",
help="Remove systematic absent reflections."
).option(
None,
"--map_to_asu",
action="store_true",
help="Map to asymmetric unit."
).option(
"--show_details_if_error",
action="store_true",
help="Show data details for some errors."
).option(
"--show_log",
action="store_true",
help="Show some output."
).option(
"--ignore_bad_sigmas",
action="store_true",
help=
"Set sigmas to None instead of raising an error when bad sigmas "
"are present."
).option(
"--extend_flags",
action="store_true",
help="Extend R-free flags to cover all reflections if necessary.")
).process(args=args)
except Exception, e:
if (str(e) != "0"): print str(e)
sys.exit(0)
def run(args, command_name="phenix.fobs_minus_fobs_map", log=None):
if (len(args) == 0): args = ["--help"]
examples = """Examples:
phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb
phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \
high_res=2.0 sigma_cutoff=2 scattering_table=neutron"""
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description=examples).option("--silent",
action="store_true",
help="Suppress output to the screen.").
enable_symmetry_comprehensive()).process(args=args)
#
if (log is None):
log = sys.stdout
if (not command_line.options.silent):
utils.print_header("phenix.fobs_minus_fobs_map", out=log)
print("Command line arguments: ", file=log)
print(args, file=log)
print(file=log)
#
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=log,
suppress_symmetry_related_errors=True)
working_phil = processed_args.params
if (not command_line.options.silent):
print("*** Parameters:", file=log)
working_phil.show(out=log)
print(file=log)
params = working_phil.extract()
consensus_symmetry = None
if (params.ignore_non_isomorphous_unit_cells):
if (None in [
params.f_obs_1_file_name, params.f_obs_2_file_name,
params.phase_source
]):
raise Sorry(
"The file parameters (f_obs_1_file_name, f_obs_2_file_name, " +
"phase_source) must be specified explicitly when " +
"ignore_non_isomorphous_unit_cells=True.")
symm_manager = iotbx.symmetry.manager()
pdb_in = iotbx.file_reader.any_file(params.phase_source,
force_type="pdb")
symm_manager.process_pdb_file(pdb_in)
hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name,
force_type="hkl")
sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1)
hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name,
force_type="hkl")
sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2)
out = StringIO()
symm_manager.show(out=out)
if (sg_err_1) or (sg_err_2):
raise Sorry((
"Incompatible space groups in input files:\n%s\nAll files " +
"must have the same point group (and ideally the same space group). "
+
"Please note that any symmetry information in the PDB file will be "
+ "used first.") % out.getvalue())
elif (uc_err_1) or (uc_err_2):
libtbx.call_back(
message="warn",
data=
("Crystal symmetry mismatch:\n%s\nCalculations will continue "
+
"using the symmetry in the PDB file (or if not available, the "
+
"first reflection file), but the maps should be treated with "
+ "extreme suspicion.") % out.getvalue())
crystal_symmetry = symm_manager.as_symmetry_object()
else:
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
suppress_symmetry_related_errors=False,
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=StringIO())
crystal_symmetry = processed_args.crystal_symmetry
#
pdb_file_names = processed_args.pdb_file_names
if (len(processed_args.pdb_file_names) == 0):
if (params.phase_source is not None):
pdb_file_names = [params.phase_source]
else:
raise Sorry("No PDB file found.")
# Extaract Fobs1, Fobs2
f_obss = []
if (len(processed_args.reflection_files) == 2):
for reflection_file in processed_args.reflection_files:
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[reflection_file],
err=null_out())
# XXX UGLY !!!
try:
parameters = utils.data_and_flags_master_params().extract()
if (params.f_obs_1_label is not None):
parameters.labels = [params.f_obs_1_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
keep_going=True,
parameters=parameters,
log=null_out())
except: # intentional
parameters = utils.data_and_flags_master_params().extract()
if (params.f_obs_2_label is not None):
parameters.labels = [params.f_obs_2_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
keep_going=True,
parameters=parameters,
log=null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
else:
if ([params.f_obs_1_file_name,
params.f_obs_2_file_name].count(None) == 2):
raise Sorry("No reflection data file found.")
for file_name, label in zip(
[params.f_obs_1_file_name, params.f_obs_2_file_name],
[params.f_obs_1_label, params.f_obs_2_label]):
reflection_file = reflection_file_reader.any_reflection_file(
file_name=file_name, ensure_read_access=False)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[reflection_file],
err=null_out())
parameters = utils.data_and_flags_master_params().extract()
if (label is not None):
parameters.labels = [label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=reflection_file_server,
parameters=parameters,
keep_going=True,
log=null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
if (len(f_obss) != 2):
raise Sorry(" ".join(errors))
if (not command_line.options.silent):
for ifobs, fobs in enumerate(f_obss):
print("*** Summary for data set %d:" % ifobs, file=log)
fobs.show_comprehensive_summary(f=log)
print(file=log)
pdb_combined = combine_unique_pdb_files(file_names=pdb_file_names)
pdb_combined.report_non_unique(out=log)
if (len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
#
raw_recs = flex.std_string()
for rec in pdb_combined.raw_records:
if (rec.upper().count("CRYST1") == 0):
raw_recs.append(rec)
raw_recs.append(
iotbx.pdb.format_cryst1_record(crystal_symmetry=crystal_symmetry))
#
pdb_in = iotbx.pdb.input(source_info=None, lines=raw_recs)
model = mmtbx.model.manager(model_input=pdb_in)
d_min = min(f_obss[0].d_min(), f_obss[1].d_min())
model.setup_scattering_dictionaries(
scattering_table=params.scattering_table, d_min=d_min)
xray_structure = model.get_xray_structure()
hierarchy = model.get_hierarchy()
#
omit_sel = flex.bool(hierarchy.atoms_size(), False)
if (params.advanced.omit_selection is not None):
print("Will omit selection from phasing model:", file=log)
print(" " + params.advanced.omit_selection, file=log)
omit_sel = hierarchy.atom_selection_cache().selection(
params.advanced.omit_selection)
print("%d atoms selected for removal" % omit_sel.count(True), file=log)
del hierarchy
xray_structure = xray_structure.select(~omit_sel)
if (not command_line.options.silent):
print("*** Model summary:", file=log)
xray_structure.show_summary(f=log)
print(file=log)
info0 = f_obss[0].info()
info1 = f_obss[1].info()
f_obss[0] = f_obss[0].resolution_filter(
d_min=params.high_resolution,
d_max=params.low_resolution).set_info(info0)
f_obss[1] = f_obss[1].resolution_filter(
d_min=params.high_resolution,
d_max=params.low_resolution).set_info(info1)
if (params.sigma_cutoff is not None):
for i in [0, 1]:
if (f_obss[i].sigmas() is not None):
sel = f_obss[i].data(
) > f_obss[i].sigmas() * params.sigma_cutoff
f_obss[i] = f_obss[i].select(sel).set_info(info0)
for k, f_obs in enumerate(f_obss):
if (f_obs.indices().size() == 0):
raise Sorry(
"No data left in array %d (labels=%s) after filtering!" %
(k + 1, f_obs.info().label_string()))
output_file_name = params.output_file
if (output_file_name is None) and (params.file_name_prefix is not None):
output_file_name = "%s_%s.mtz" % (params.file_name_prefix,
params.job_id)
output_files = compute_fo_minus_fo_map(
data_arrays=f_obss,
xray_structure=xray_structure,
log=log,
silent=command_line.options.silent,
output_file=output_file_name,
peak_search=params.find_peaks_holes,
map_cutoff=params.map_cutoff,
peak_search_params=params.peak_search,
multiscale=params.advanced.multiscale,
anomalous=params.advanced.anomalous).file_names
return output_files
def run(args, command_name = "phenix.fobs_minus_fobs_map", log=None):
if(len(args) == 0): args = ["--help"]
examples = """Examples:
phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb
phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \
f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \
high_res=2.0 sigma_cutoff=2 scattering_table=neutron"""
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description=examples)
.option("--silent",
action="store_true",
help="Suppress output to the screen.")
.enable_symmetry_comprehensive()
).process(args=args)
#
if (log is None) :
log = sys.stdout
if(not command_line.options.silent):
utils.print_header("phenix.fobs_minus_fobs_map", out = log)
print >> log, "Command line arguments: "
print >> log, args
print >> log
#
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=log,
suppress_symmetry_related_errors=True)
working_phil = processed_args.params
if(not command_line.options.silent):
print >> log, "*** Parameters:"
working_phil.show(out = log)
print >> log
params = working_phil.extract()
consensus_symmetry = None
if (params.ignore_non_isomorphous_unit_cells) :
if (None in [params.f_obs_1_file_name, params.f_obs_2_file_name,
params.phase_source]):
raise Sorry("The file parameters (f_obs_1_file_name, f_obs_2_file_name, "+
"phase_source) must be specified explicitly when "+
"ignore_non_isomorphous_unit_cells=True.")
symm_manager = iotbx.symmetry.manager()
pdb_in = iotbx.file_reader.any_file(params.phase_source, force_type="pdb")
symm_manager.process_pdb_file(pdb_in)
hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name,
force_type="hkl")
sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1)
hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name,
force_type="hkl")
sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2)
out = StringIO()
symm_manager.show(out=out)
if (sg_err_1) or (sg_err_2) :
raise Sorry(("Incompatible space groups in input files:\n%s\nAll files "+
"must have the same point group (and ideally the same space group). "+
"Please note that any symmetry information in the PDB file will be "+
"used first.") % out.getvalue())
elif (uc_err_1) or (uc_err_2) :
libtbx.call_back(message="warn",
data=("Crystal symmetry mismatch:\n%s\nCalculations will continue "+
"using the symmetry in the PDB file (or if not available, the "+
"first reflection file), but the maps should be treated with "+
"extreme suspicion.") % out.getvalue())
crystal_symmetry = symm_manager.as_symmetry_object()
else :
processed_args = utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=fo_minus_fo_master_params(),
absolute_angle_tolerance=5,
absolute_length_tolerance=1,
log=StringIO())
crystal_symmetry = processed_args.crystal_symmetry
#
pdb_file_names = processed_args.pdb_file_names
if(len(processed_args.pdb_file_names) == 0):
if(params.phase_source is not None):
pdb_file_names = [params.phase_source]
else:
raise Sorry("No PDB file found.")
# Extaract Fobs1, Fobs2
f_obss = []
if(len(processed_args.reflection_files)==2):
for reflection_file in processed_args.reflection_files:
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = [reflection_file],
err = null_out())
# XXX UGLY !!!
try:
parameters = utils.data_and_flags_master_params().extract()
if(params.f_obs_1_label is not None):
parameters.labels = [params.f_obs_1_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
keep_going = True,
parameters = parameters,
log = null_out())
except: # intentional
parameters = utils.data_and_flags_master_params().extract()
if(params.f_obs_2_label is not None):
parameters.labels = [params.f_obs_2_label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
keep_going = True,
parameters = parameters,
log = null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
else:
if([params.f_obs_1_file_name,params.f_obs_2_file_name].count(None)==2):
raise Sorry("No reflection data file found.")
for file_name, label in zip([params.f_obs_1_file_name,params.f_obs_2_file_name],
[params.f_obs_1_label,params.f_obs_2_label]):
reflection_file = reflection_file_reader.any_reflection_file(
file_name = file_name, ensure_read_access = False)
reflection_file_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = [reflection_file],
err = null_out())
parameters = utils.data_and_flags_master_params().extract()
if(label is not None):
parameters.labels = [label]
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = reflection_file_server,
parameters = parameters,
keep_going = True,
log = null_out())
f_obss.append(determine_data_and_flags_result.f_obs)
if(len(f_obss)!=2):
raise Sorry(" ".join(errors))
if(not command_line.options.silent):
for ifobs, fobs in enumerate(f_obss):
print >> log, "*** Summary for data set %d:"%ifobs
fobs.show_comprehensive_summary(f = log)
print >> log
pdb_combined = combine_unique_pdb_files(file_names = pdb_file_names)
pdb_combined.report_non_unique(out = log)
if(len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
#
raw_recs = flex.std_string()
for rec in pdb_combined.raw_records:
if(rec.upper().count("CRYST1")==0):
raw_recs.append(rec)
raw_recs.append(iotbx.pdb.format_cryst1_record(
crystal_symmetry = crystal_symmetry))
#
pdb_in = iotbx.pdb.input(source_info = None, lines = raw_recs)
hierarchy = pdb_in.construct_hierarchy()
omit_sel = flex.bool(hierarchy.atoms_size(), False)
if (params.advanced.omit_selection is not None) :
print >> log, "Will omit selection from phasing model:"
print >> log, " " + params.advanced.omit_selection
omit_sel = hierarchy.atom_selection_cache().selection(
params.advanced.omit_selection)
print >> log, "%d atoms selected for removal" % omit_sel.count(True)
xray_structure = pdb_in.xray_structure_simple()
xray_structure = xray_structure.select(~omit_sel)
if(not command_line.options.silent):
print >> log, "*** Model summary:"
xray_structure.show_summary(f = log)
print >> log
info0 = f_obss[0].info()
info1 = f_obss[1].info()
f_obss[0] = f_obss[0].resolution_filter(d_min = params.high_resolution,
d_max = params.low_resolution).set_info(info0)
f_obss[1] = f_obss[1].resolution_filter(d_min = params.high_resolution,
d_max = params.low_resolution).set_info(info1)
if(params.sigma_cutoff is not None):
for i in [0,1]:
if(f_obss[i].sigmas() is not None):
sel = f_obss[i].data() > f_obss[i].sigmas()*params.sigma_cutoff
f_obss[i] = f_obss[i].select(sel).set_info(info0)
for k, f_obs in enumerate(f_obss) :
if (f_obs.indices().size() == 0) :
raise Sorry("No data left in array %d (labels=%s) after filtering!" % (k+1,
f_obs.info().label_string()))
output_file_name = params.output_file
if (output_file_name is None) and (params.file_name_prefix is not None) :
output_file_name = "%s_%s.mtz" % (params.file_name_prefix, params.job_id)
output_files = compute_fo_minus_fo_map(
data_arrays = f_obss,
xray_structure = xray_structure,
log = log,
silent = command_line.options.silent,
output_file = output_file_name,
peak_search=params.find_peaks_holes,
map_cutoff=params.map_cutoff,
peak_search_params=params.peak_search,
multiscale=params.advanced.multiscale,
anomalous=params.advanced.anomalous).file_names
return output_files
def run(args, command_name):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_line = (iotbx_option_parser(
usage=command_name+" [options] [cod_id...]")
.enable_chunk(easy_all=True)
.enable_multiprocessing()
.option(None, "--parse_only",
action="store_true")
.option(None, "--cif_only",
action="store_true")
.option(None, "--hkl_only",
action="store_true")
.option("-v", "--verbose",
action="store_true")
).process(args=args)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=[command_name, __file__])):
show_times()
return
co = command_line.options
cod_ids = command_line.args
assert [co.cif_only, co.hkl_only].count(True) <= 1
if co.cif_only: ext = "cif"
elif co.hkl_only: ext = "hkl"
else: ext = None
verbose = co.verbose
parse_only = co.parse_only
#
cod_hkl_cif = cod_tools.build_hkl_cif(cod_ids=cod_ids, ext=ext)
cod_hkl_cif.show_summary()
hkl_files = cod_hkl_cif.hkl
cif_files = cod_hkl_cif.cif
#
n_total = 0
#
parsing_errors = {}
build_errors = {}
ignored_errors = {}
skipped = set()
#
files_to_parse = []
files_to_parse.extend(hkl_files.values())
files_to_parse.extend(cif_files.values())
for i, path in enumerate(files_to_parse):
n_total += 1
if (i % command_line.chunk.n != command_line.chunk.i): continue
try:
cod_id = os.path.basename(path)
cif_obj = iotbx.cif.reader(file_path=path)
if parse_only: continue
skip_file = False
for cif_block in cif_obj.model().values():
value = cif_block.get("_cod_error_flag")
keys = set(cif_block.keys())
if (value in ["errors", "retracted"]):
skip_file = True
skipped.add(cod_id)
if verbose:
print "SKIPPING: _cod_error_flag %s: %s" % (value, cod_id)
elif (len(set([
"_space_group_symop_ssg_operation_algebraic",
"_space_group_ssg_name"]).intersection(keys)) != 0):
skipped.add(cod_id)
if verbose:
print "SKIPPING: COD entry with super-space group:", cod_id
elif (len(set([
"_refln_index_m",
"_refln_index_m_1"]).intersection(keys)) != 0):
if verbose:
print "SKIPPING: COD entry with _refln_index_m:", cod_id
skipped.add(cod_id)
if skip_file: continue
if path.endswith('.cif'):
cif_obj.build_crystal_structures()
elif path.endswith('.hkl'):
cif_obj.build_miller_arrays()
else:
iotbx.cif.cctbx_data_structures_from_cif(cif_model=cif_obj.model())
except KeyboardInterrupt:
print "CAUGHT EXCEPTION: KeyboardInterrupt"
return
except CifBuilderError, e:
e_str = str(e)
if not verbose and (
e_str.startswith("No atomic coordinates could be found") or
e_str.startswith(
"No symmetry instructions could be extracted from the cif block")):
ignored_errors.setdefault(cod_id, e_str)
continue
sys.stdout.flush()
print >> sys.stderr, \
"CAUGHT EXCEPTION: %s: %s: %s" % (command_name, cod_id, str(e))
if verbose:
traceback.print_exc()
print >> sys.stderr
build_errors.setdefault(cod_id, e_str)
sys.stderr.flush()
except CifParserError, e:
sys.stdout.flush()
e_str = str(e)
parsing_errors.setdefault(cod_id, e_str)
print >> sys.stderr, \
"PARSING ERROR: %s: %s: %s" % (command_name, cod_id, e_str)
if verbose:
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
def run(args, command_name = "mmtbx.csv_to_mtz"):
if (len(args) == 0): args = ["--help"]
try:
command_line = (iotbx_option_parser(
usage="%s [reflection_csv_file] [options]" % command_name,
description='Example: %s 1m5u-sf.csv --use_model=1m5u.pdb'%command_name)
.enable_symmetry_comprehensive()
.option(None, "--output_file",
action="store",
default=False,
type="string",
help="Output mtz file name.")
.option(None, "--use_model",
action="store",
default=False,
type="string",
help="Use PDB model to make better guess about reflection data type.")
).process(args=args)
except Exception as e:
if(str(e) != "0"): print(str(e))
sys.exit(0)
crystal_symmetry = command_line.symmetry
if(command_line.symmetry.unit_cell() is None or
command_line.symmetry.space_group_info() is None):
if(command_line.options.use_model):
crystal_symmetry = crystal_symmetry_from_pdb.extract_from(
file_name=command_line.options.use_model)
if(crystal_symmetry.unit_cell() is None or
crystal_symmetry.space_group_info() is None):
raise Sorry(
"Crystal symmetry is not defined. Please use the --symmetry option.\n"
"Type %s without arguments to see more options."%command_name)
if(len(command_line.args) > 1):
print("%d arguments are given from the command line:"% \
len(command_line.args), command_line.args)
raise Sorry("Please specify one reflection csv file.")
file_name = command_line.args[0]
if(not os.path.isfile(file_name)):
raise Sorry("File is not found: %s"%file_name)
data = flex.double()
sigmas = flex.double()
flags = flex.int()
column_ids, columns = parse_csv_file(file_name=file_name)
data_label_root = column_ids[3]
ms = miller.set(crystal_symmetry, flex.miller_index(columns[0]))
for d in columns[1]:
data.append(float(d))
for sig in columns[2]:
sigmas.append(float(sig))
for flag in columns[3]:
flags.append(int(flag))
assert len(data) == len(sigmas)
assert len(flags) == len(data)
ma = miller.array(ms, data, sigmas)
if data_label_root.startswith('F'):
ma.set_observation_type_xray_amplitude()
elif data_label_root.startswith('I'):
ma.set_observation_type_xray_intensity()
else:
ma.set_observation_type_xray_amplitude()
flags_ma = miller.set(
crystal_symmetry = crystal_symmetry,
indices = ma.indices()).array(data = flags)
mtz_dataset = ma.as_mtz_dataset(
column_root_label = data_label_root)
mtz_dataset.add_miller_array(
miller_array = flags_ma,
column_root_label = "R-free-flags")
mtz_object = mtz_dataset.mtz_object()
mtz_object.write(file_name = command_line.options.output_file)
def run(args,
command_name="phenix.cif_as_mtz",
out=sys.stdout,
return_as_miller_arrays=False):
if (len(args) == 0): args = ["--help"]
try:
command_line = (iotbx_option_parser(
usage="%s [reflection_cif_file] [options]" % command_name,
description='Example: %s r1o9ksf.ent --symmetry=pdb1o9k.ent' %
command_name
).enable_symmetry_comprehensive().option(
None,
"--output_file_name",
action="store",
default=False,
type="string",
help="Output mtz file name."
).option(
None,
"--wavelength_id",
action="store",
default=None,
type="int",
help="Extract data set with given wavelength_id."
).option(
None,
"--crystal_id",
action="store",
default=None,
type="int",
help="Extract data set with given crystal_id."
).option(
None,
"--output_r_free_label",
action="store",
default="R-free-flags",
type="string",
help=
"MTZ column label to use for R-free flags (default: R-free-flags)"
).option(
None,
"--merge",
action="store_true",
help="Merge non-unique data where present."
).option(
None,
"--incompatible_flags_to_work_set",
action="store_true",
help=
"When merging place reflections with incompatible flags into the "
"working set."
).option(
None,
"--remove_systematic_absences",
action="store_true",
help="Remove systematic absent reflections."
).option(
None,
"--map_to_asu",
action="store_true",
help="Map to asymmetric unit."
).option(
"--show_details_if_error",
action="store_true",
help="Show data details for some errors."
).option(
"--show_log",
action="store_true",
help="Show some output."
).option(
"--ignore_bad_sigmas",
action="store_true",
help=
"Set sigmas to None instead of raising an error when bad sigmas "
"are present."
).option(
"--extend_flags",
action="store_true",
help="Extend R-free flags to cover all reflections if necessary.")
).process(args=args)
except Exception as e:
if (str(e) != "0"): print(str(e))
sys.exit(0)
crystal_symmetry = command_line.symmetry
if (len(command_line.args) > 1):
print("%d arguments are given from the command line:"% \
len(command_line.args), command_line.args, file=out)
raise Sorry("Please specify one reflection cif file.")
file_name = command_line.args[0]
if (not os.path.isfile(file_name)):
raise Sorry("File is not found: %s" % file_name)
output_r_free_label = command_line.options.output_r_free_label
if ((not output_r_free_label[0] in string.ascii_uppercase)
or (re.search(r"[^a-zA-Z0-9_\-]", output_r_free_label))):
raise Sorry((
"%s is not a suitable column label. MTZ format requires " +
"an uppercase letter as the first character, and only alphanumeric "
+ "characters or hyphens in the rest of the string.") %
output_r_free_label)
result = process_files(
file_name=file_name,
crystal_symmetry=crystal_symmetry,
output_file_name=command_line.options.output_file_name,
wavelength_id=command_line.options.wavelength_id,
crystal_id=command_line.options.crystal_id,
show_details_if_error=command_line.options.show_details_if_error,
output_r_free_label=command_line.options.output_r_free_label,
merge_non_unique_under_symmetry=command_line.options.merge,
map_to_asu=command_line.options.map_to_asu,
remove_systematic_absences=command_line.options.
remove_systematic_absences,
incompatible_flags_to_work_set=command_line.options.
incompatible_flags_to_work_set,
return_as_miller_arrays=return_as_miller_arrays,
ignore_bad_sigmas=command_line.options.ignore_bad_sigmas,
extend_flags=command_line.options.extend_flags,
log=out)
if return_as_miller_arrays:
return result
def run(args, command_name):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_line = (iotbx_option_parser(
usage=command_name + " [options] [cod_id...]").enable_chunk(
easy_all=True).enable_multiprocessing().option(
None, "--parse_only", action="store_true").option(
None, "--cif_only", action="store_true").option(
None, "--hkl_only", action="store_true").option(
"-v", "--verbose",
action="store_true")).process(args=args)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=[command_name, __file__])):
show_times()
return
co = command_line.options
cod_ids = command_line.args
assert [co.cif_only, co.hkl_only].count(True) <= 1
if co.cif_only: ext = "cif"
elif co.hkl_only: ext = "hkl"
else: ext = None
verbose = co.verbose
parse_only = co.parse_only
#
cod_hkl_cif = cod_tools.build_hkl_cif(cod_ids=cod_ids, ext=ext)
cod_hkl_cif.show_summary()
hkl_files = cod_hkl_cif.hkl
cif_files = cod_hkl_cif.cif
#
n_total = 0
#
parsing_errors = {}
build_errors = {}
ignored_errors = {}
skipped = set()
#
files_to_parse = []
files_to_parse.extend(hkl_files.values())
files_to_parse.extend(cif_files.values())
for i, path in enumerate(files_to_parse):
n_total += 1
if (i % command_line.chunk.n != command_line.chunk.i): continue
try:
cod_id = os.path.basename(path)
cif_obj = iotbx.cif.reader(file_path=path)
if parse_only: continue
skip_file = False
for cif_block in cif_obj.model().values():
value = cif_block.get("_cod_error_flag")
keys = set(cif_block.keys())
if (value in ["errors", "retracted"]):
skip_file = True
skipped.add(cod_id)
if verbose:
print("SKIPPING: _cod_error_flag %s: %s" %
(value, cod_id))
elif (len(
set([
"_space_group_symop_ssg_operation_algebraic",
"_space_group_ssg_name"
]).intersection(keys)) != 0):
skipped.add(cod_id)
if verbose:
print("SKIPPING: COD entry with super-space group:",
cod_id)
elif (len(
set(["_refln_index_m", "_refln_index_m_1"
]).intersection(keys)) != 0):
if verbose:
print("SKIPPING: COD entry with _refln_index_m:",
cod_id)
skipped.add(cod_id)
if skip_file: continue
if path.endswith('.cif'):
cif_obj.build_crystal_structures()
elif path.endswith('.hkl'):
cif_obj.build_miller_arrays()
else:
iotbx.cif.cctbx_data_structures_from_cif(
cif_model=cif_obj.model())
except KeyboardInterrupt:
print("CAUGHT EXCEPTION: KeyboardInterrupt")
return
except CifBuilderError as e:
e_str = str(e)
if not verbose and (e_str.startswith(
"No atomic coordinates could be found"
) or e_str.startswith(
"No symmetry instructions could be extracted from the cif block"
)):
ignored_errors.setdefault(cod_id, e_str)
continue
sys.stdout.flush()
print("CAUGHT EXCEPTION: %s: %s: %s" %
(command_name, cod_id, str(e)),
file=sys.stderr)
if verbose:
traceback.print_exc()
print(file=sys.stderr)
build_errors.setdefault(cod_id, e_str)
sys.stderr.flush()
except CifParserError as e:
sys.stdout.flush()
e_str = str(e)
parsing_errors.setdefault(cod_id, e_str)
print("PARSING ERROR: %s: %s: %s" % (command_name, cod_id, e_str),
file=sys.stderr)
if verbose:
traceback.print_exc()
print(file=sys.stderr)
sys.stderr.flush()
except Exception as e:
sys.stdout.flush()
e_str = str(e)
build_errors.setdefault(cod_id, e_str)
print("CAUGHT EXCEPTION: %s: %s: %s" %
(command_name, cod_id, e_str),
file=sys.stderr)
if verbose:
traceback.print_exc()
print(file=sys.stderr)
sys.stderr.flush()
print()
print("Number successfully parsed: %i/%i" \
% (n_total-len(parsing_errors),n_total))
if not parse_only:
print("Number skipped:", len(skipped))
print("Number of exceptions caught:", len(build_errors))
print("Number of exceptions ignored:", len(ignored_errors))
print()
#
show_times()
result = group_args(n_hkl=len(hkl_files),
n_cif=len(cif_files),
n_hkl_cif_pairs=len(cod_hkl_cif.hkl_cif_pairs),
parsing_errors=parsing_errors,
build_errors=build_errors,
ignored_errors=ignored_errors,
skipped=skipped)
easy_pickle.dump("result_%03i.pickle" % command_line.chunk.i, result)
print()
def run(self, args, command_name, out=sys.stdout):
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description='Example: %s data.mtz data.mtz ref_model.pdb'%command_name)
.option(None, "--show_defaults",
action="store_true",
help="Show list of parameters.")
).process(args=args)
cif_file = None
processed_args = utils.process_command_line_args(
args = args,
log = sys.stdout,
master_params = master_phil)
params = processed_args.params
if(params is None): params = master_phil
self.params = params.extract().ensemble_probability
pdb_file_names = processed_args.pdb_file_names
if len(pdb_file_names) != 1 :
raise Sorry("Only one PDB structure may be used")
pdb_file = file_reader.any_file(pdb_file_names[0])
self.log = multi_out()
self.log.register(label="stdout", file_object=sys.stdout)
self.log.register(
label="log_buffer",
file_object=StringIO(),
atexit_send_to=None)
sys.stderr = self.log
log_file = open(pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.log', "w")
self.log.replace_stringio(
old_label="log_buffer",
new_label="log",
new_file_object=log_file)
utils.print_header(command_name, out = self.log)
params.show(out = self.log)
#
f_obs = None
r_free_flags = None
reflection_files = processed_args.reflection_files
if self.params.fobs_vs_fcalc_post_nll:
if len(reflection_files) == 0:
raise Sorry("Fobs from input MTZ required for fobs_vs_fcalc_post_nll")
if len(reflection_files) > 0:
crystal_symmetry = processed_args.crystal_symmetry
print >> self.log, 'Reflection file : ', processed_args.reflection_file_names[0]
utils.print_header("Model and data statistics", out = self.log)
rfs = reflection_file_server(
crystal_symmetry = crystal_symmetry,
reflection_files = processed_args.reflection_files,
log = self.log)
parameters = utils.data_and_flags_master_params().extract()
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server = rfs,
parameters = parameters,
data_parameter_scope = "refinement.input.xray_data",
flags_parameter_scope = "refinement.input.xray_data.r_free_flags",
data_description = "X-ray data",
keep_going = True,
log = self.log)
f_obs = determine_data_and_flags_result.f_obs
number_of_reflections = f_obs.indices().size()
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if(r_free_flags is None):
r_free_flags=f_obs.array(data=flex.bool(f_obs.data().size(), False))
# process PDB
pdb_file.assert_file_type("pdb")
#
pdb_in = hierarchy.input(file_name=pdb_file.file_name)
ens_pdb_hierarchy = pdb_in.construct_hierarchy()
ens_pdb_hierarchy.atoms().reset_i_seq()
ens_pdb_xrs_s = pdb_in.input.xray_structures_simple()
number_structures = len(ens_pdb_xrs_s)
print >> self.log, 'Number of structure in ensemble : ', number_structures
# Calculate sigmas from input map only
if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None:
# process MTZ
input_file = file_reader.any_file(self.params.ensemble_sigma_map_input)
if input_file.file_type == "hkl" :
if input_file.file_object.file_type() != "ccp4_mtz" :
raise Sorry("Only MTZ format accepted for map input")
else:
mtz_file = input_file
else:
raise Sorry("Only MTZ format accepted for map input")
miller_arrays = mtz_file.file_server.miller_arrays
map_coeffs_1 = miller_arrays[0]
#
xrs_list = []
for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
# get sigma levels from ensemble fc for each structure
xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy,
ens_pdb_xrs = ens_pdb_xrs,
map_coeffs_1 = map_coeffs_1,
residue_detail = self.params.residue_detail,
ignore_hd = self.params.ignore_hd,
log = self.log)
xrs_list.append(xrs)
# write ensemble pdb file, occupancies as sigma level
filename = pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_vs_' + self.params.ensemble_sigma_map_input.replace('.mtz','') + '_pensemble.pdb'
write_ensemble_pdb(filename = filename,
xrs_list = xrs_list,
ens_pdb_hierarchy = ens_pdb_hierarchy
)
# Do full analysis vs Fobs
else:
model_map_coeffs = []
fmodel = None
# Get <fcalc>
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
ens_pdb_xrs.set_occupancies(1.0)
if model == 0:
# If mtz not supplied get fobs from xray structure...
# Use input Fobs for scoring against nll
if self.params.fobs_vs_fcalc_post_nll:
dummy_fobs = f_obs
else:
if f_obs == None:
if self.params.fcalc_high_resolution == None:
raise Sorry("Please supply high resolution limit or input mtz file.")
dummy_dmin = self.params.fcalc_high_resolution
dummy_dmax = self.params.fcalc_low_resolution
else:
print >> self.log, 'Supplied mtz used to determine high and low resolution cuttoffs'
dummy_dmax, dummy_dmin = f_obs.d_max_min()
#
dummy_fobs = abs(ens_pdb_xrs.structure_factors(d_min = dummy_dmin).f_calc())
dummy_fobs.set_observation_type_xray_amplitude()
# If mtz supplied, free flags are over written to prevent array size error
r_free_flags = dummy_fobs.array(data=flex.bool(dummy_fobs.data().size(),False))
#
fmodel = utils.fmodel_simple(
scattering_table = "wk1995",
xray_structures = [ens_pdb_xrs],
f_obs = dummy_fobs,
target_name = 'ls',
bulk_solvent_and_scaling = False,
r_free_flags = r_free_flags
)
f_calc_ave = fmodel.f_calc().array(data = fmodel.f_calc().data()*0).deep_copy()
# XXX Important to ensure scale is identical for each model and <model>
fmodel.set_scale_switch = 1.0
f_calc_ave_total = fmodel.f_calc().data().deep_copy()
else:
fmodel.update_xray_structure(xray_structure = ens_pdb_xrs,
update_f_calc = True,
update_f_mask = False)
f_calc_ave_total += fmodel.f_calc().data().deep_copy()
print >> self.log, 'Model :', model+1
print >> self.log, "\nStructure vs real Fobs (no bulk solvent or scaling)"
print >> self.log, 'Rwork : %5.4f '%fmodel.r_work()
print >> self.log, 'Rfree : %5.4f '%fmodel.r_free()
print >> self.log, 'K1 : %5.4f '%fmodel.scale_k1()
fcalc_edm = fmodel.electron_density_map()
fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type = 'Fc')
fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset(column_root_label ='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_mtz_dataset.mtz_object().write(file_name = str(model+1)+"_Fc.mtz")
model_map_coeffs.append(fcalc_map_coeffs.deep_copy())
fmodel.update(f_calc = f_calc_ave.array(f_calc_ave_total / number_structures))
print >> self.log, "\nEnsemble vs real Fobs (no bulk solvent or scaling)"
print >> self.log, 'Rwork : %5.4f '%fmodel.r_work()
print >> self.log, 'Rfree : %5.4f '%fmodel.r_free()
print >> self.log, 'K1 : %5.4f '%fmodel.scale_k1()
# Get <Fcalc> map
fcalc_ave_edm = fmodel.electron_density_map()
fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients(map_type = 'Fc').deep_copy()
fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset(column_root_label ='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_ave_mtz_dataset.mtz_object().write(file_name = "aveFc.mtz")
fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map()
fcalc_ave_map_coeffs.apply_volume_scaling()
fcalc_ave_map_data = fcalc_ave_map_coeffs.real_map_unpadded()
fcalc_ave_map_stats = maptbx.statistics(fcalc_ave_map_data)
print >> self.log, "<Fcalc> Map Stats :"
fcalc_ave_map_stats.show_summary(f = self.log)
offset = fcalc_ave_map_stats.min()
model_neg_ll = []
number_previous_scatters = 0
# Run through structure list again and get probability
xrs_list = []
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
if self.params.verbose:
print >> self.log, '\n\nModel : ', model+1
# Get model atom sigmas vs Fcalc
fcalc_map = model_map_coeffs[model].fft_map()
fcalc_map.apply_volume_scaling()
fcalc_map_data = fcalc_map.real_map_unpadded()
fcalc_map_stats = maptbx.statistics(fcalc_map_data)
if self.params.verbose:
print >> self.log, "Fcalc map stats :"
fcalc_map_stats.show_summary(f = self.log)
xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy,
ens_pdb_xrs = ens_pdb_xrs,
fft_map_1 = fcalc_map,
model_i = model,
residue_detail = self.params.residue_detail,
ignore_hd = self.params.ignore_hd,
number_previous_scatters = number_previous_scatters,
log = self.log)
fcalc_sigmas = xrs.scatterers().extract_occupancies()
del fcalc_map
# Get model atom sigmas vs <Fcalc>
xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy,
ens_pdb_xrs = ens_pdb_xrs,
fft_map_1 = fcalc_ave_map_coeffs,
model_i = model,
residue_detail = self.params.residue_detail,
ignore_hd = self.params.ignore_hd,
number_previous_scatters = number_previous_scatters,
log = self.log)
### For testing other residue averaging options
#print xrs.residue_selections
fcalc_ave_sigmas = xrs.scatterers().extract_occupancies()
# Probability of model given <model>
prob = fcalc_ave_sigmas / fcalc_sigmas
# XXX debug option
if False:
for n,p in enumerate(prob):
print >> self.log, ' {0:5d} {1:5.3f}'.format(n,p)
# Set probabilty between 0 and 1
# XXX Make Histogram / more stats
prob_lss_zero = flex.bool(prob <= 0)
prob_grt_one = flex.bool(prob > 1)
prob.set_selected(prob_lss_zero, 0.001)
prob.set_selected(prob_grt_one, 1.0)
xrs.set_occupancies(prob)
xrs_list.append(xrs)
sum_neg_ll = sum(-flex.log(prob))
model_neg_ll.append((sum_neg_ll, model))
if self.params.verbose:
print >> self.log, 'Model probability stats :'
print >> self.log, prob.min_max_mean().show()
print >> self.log, ' Count < 0.0 : ', prob_lss_zero.count(True)
print >> self.log, ' Count > 1.0 : ', prob_grt_one.count(True)
# For averaging by residue
number_previous_scatters += ens_pdb_xrs.sites_cart().size()
# write ensemble pdb file, occupancies as sigma level
write_ensemble_pdb(filename = pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.pdb',
xrs_list = xrs_list,
ens_pdb_hierarchy = ens_pdb_hierarchy
)
# XXX Test ordering models by nll
# XXX Test removing nth percentile atoms
if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll:
for percentile in [1.0,0.975,0.95,0.9,0.8,0.6,0.2]:
model_neg_ll = sorted(model_neg_ll)
f_calc_ave_total_reordered = None
print_list = []
for i_neg_ll in model_neg_ll:
xrs = xrs_list[i_neg_ll[1]]
nll_occ = xrs.scatterers().extract_occupancies()
# Set q=0 nth percentile atoms
sorted_nll_occ = sorted(nll_occ, reverse=True)
number_atoms = len(sorted_nll_occ)
percentile_prob_cutoff = sorted_nll_occ[int(number_atoms * percentile)-1]
cutoff_selections = flex.bool(nll_occ < percentile_prob_cutoff)
cutoff_nll_occ = flex.double(nll_occ.size(), 1.0).set_selected(cutoff_selections, 0.0)
#XXX Debug
if False:
print '\nDebug'
for x in xrange(len(cutoff_selections)):
print cutoff_selections[x], nll_occ[x], cutoff_nll_occ[x]
print percentile
print percentile_prob_cutoff
print cutoff_selections.count(True)
print cutoff_selections.size()
print cutoff_nll_occ.count(0.0)
print 'Count q = 1 : ', cutoff_nll_occ.count(1.0)
print 'Count scatterers size : ', cutoff_nll_occ.size()
xrs.set_occupancies(cutoff_nll_occ)
fmodel.update_xray_structure(xray_structure = xrs,
update_f_calc = True,
update_f_mask = True)
if f_calc_ave_total_reordered == None:
f_calc_ave_total_reordered = fmodel.f_calc().data().deep_copy()
f_mask_ave_total_reordered = fmodel.f_masks()[0].data().deep_copy()
cntr = 1
else:
f_calc_ave_total_reordered += fmodel.f_calc().data().deep_copy()
f_mask_ave_total_reordered += fmodel.f_masks()[0].data().deep_copy()
cntr+=1
fmodel.update(f_calc = f_calc_ave.array(f_calc_ave_total_reordered / cntr).deep_copy(),
f_mask = f_calc_ave.array(f_mask_ave_total_reordered / cntr).deep_copy()
)
# Update solvent and scale
# XXX Will need to apply_back_trace on latest version
fmodel.set_scale_switch = 0
fmodel.update_all_scales()
# Reset occ for outout
xrs.set_occupancies(nll_occ)
# k1 updated vs Fobs
if self.params.fobs_vs_fcalc_post_nll:
print_list.append([cntr, i_neg_ll[0], i_neg_ll[1], fmodel.r_work(), fmodel.r_free()])
# Order models by nll and print summary
print >> self.log, '\nModels ranked by nll <Fcalc> R-factors recalculated'
print >> self.log, 'Percentile cutoff : {0:5.3f}'.format(percentile)
xrs_list_sorted_nll = []
print >> self.log, ' | NLL <Rw> <Rf> Ens Model'
for info in print_list:
print >> self.log, ' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'.format(
info[0],
info[1],
info[3],
info[4],
info[2]+1,
)
xrs_list_sorted_nll.append(xrs_list[info[2]])
# Output nll ordered ensemble
write_ensemble_pdb(filename = 'nll_ordered_' + pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.pdb',
xrs_list = xrs_list_sorted_nll,
ens_pdb_hierarchy = ens_pdb_hierarchy
)
def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) + " [options] directory|file...")
.enable_chunk(easy_all=True)
.enable_multiprocessing()
).process(args=args, min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print "TIME BEGIN cod_refine:", date_and_time()
print
#
master_phil = get_master_phil()
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print
params = work_phil.extract()
#
qi_dict = {}
all_pickles = []
for arg in remaining_args:
if (op.isdir(arg)):
for node in sorted(os.listdir(arg)):
if (node.endswith(".pickle")):
all_pickles.append(op.join(arg, node))
elif (node.startswith("qi_") and len(node) == 10):
qi = open(op.join(arg, node)).read().splitlines()
if (len(qi) == 1):
cod_id = node[3:]
quick_info = eval(qi[0])
assert cod_id not in qi_dict
qi_dict[cod_id] = quick_info
elif (op.isfile(arg)):
all_pickles.append(arg)
else:
raise RuntimeError("Not a file or directory: %s" % arg)
print "Number of pickle files:", len(all_pickles)
print "Number of quick_infos:", len(qi_dict)
sort_choice = params.sorting_of_pickle_files
if (len(qi_dict) != 0 and sort_choice is not None):
print "Sorting pickle files by n_atoms * n_refl:", sort_choice
assert sort_choice in ["down", "up"]
def sort_pickle_files():
if (sort_choice == "down"): i_sign = -1
else: i_sign = 1
buffer = []
for i,path in enumerate(all_pickles):
cod_id = op.basename(path).split(".",1)[0]
qi = qi_dict.get(cod_id)
if (qi is None): nn = 2**31
else: nn = qi[0] * qi[1] * qi[2]
buffer.append((nn, i_sign*i, path))
buffer.sort()
if (i_sign < 0):
buffer.reverse()
result = []
for elem in buffer:
result.append(elem[-1])
return result
all_pickles = sort_pickle_files()
print
#
rss = params.random_subset.size
if (rss is not None and rss > 0):
seed = params.random_subset.seed
print "Selecting subset of %d pickle files using random seed %d" % (
rss, seed)
mt = flex.mersenne_twister(seed=seed)
perm = mt.random_permutation(size=len(all_pickles))[:rss]
flags = flex.bool(len(all_pickles), False).set_selected(perm, True)
all_pickles = flex.select(all_pickles, permutation=flags.iselection())
print
#
from libtbx.path import makedirs_race
if (params.wdir_root is not None):
makedirs_race(path=params.wdir_root)
if (params.pickle_refined_dir is not None):
makedirs_race(path=params.pickle_refined_dir)
#
n_caught = 0
for i_pickle,pickle_file_name in enumerate(all_pickles):
if (i_pickle % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, pickle_file_name)
except KeyboardInterrupt:
print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt"
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s" % pickle_file_name
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
print "done_with: %s (%.2f seconds)" % (pickle_file_name, tm.elapsed())
print
sys.stdout.flush()
print
print "Number of exceptions caught:", n_caught
#
show_times()
print
print "TIME END cod_refine:", date_and_time()
def run(args, command_name):
from iotbx.option_parser import option_parser as iotbx_option_parser
from libtbx import easy_pickle
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_line = (
iotbx_option_parser(usage=command_name + " [options] [cod_id...]")
.enable_chunk(easy_all=True)
.enable_multiprocessing()
.option(None, "--max_atoms", type="int", default=99, metavar="INT")
.option(None, "--min_distance", type="float", default=0.5, metavar="FLOAT")
.option(None, "--i_obs_i_calc_plot", action="store_true", default=False)
.option(None, "--f_obs_f_calc_plot", action="store_true", default=False)
.option(None, "--max_fan_outlier_fraction", type="float", default=0.05, metavar="FLOAT")
.option(None, "--fan_offset_low", type="float", default=0.05, metavar="FLOAT")
.option(None, "--fan_offset_high", type="float", default=0.10, metavar="FLOAT")
.option(None, "--min_f_obs_f_calc_correlation", type="float", default=0.99, metavar="FLOAT")
.option(None, "--at_least_one_special_position", action="store_true", default=False)
.option(None, "--pickle_dir", type="str", default="cod_ma_xs", metavar="PATH")
).process(args=args)
if command_line.run_multiprocessing_chunks_if_applicable(command_call=[command_name]):
show_times()
return
co = command_line.options
#
from iotbx.cif import cod_tools
hkl_cif = cod_tools.build_hkl_cif(cod_ids=command_line.args)
hkl_cif.show_summary()
#
pickle_dir = co.pickle_dir
if co.at_least_one_special_position:
pickle_dir += "_special"
if not op.isdir(pickle_dir):
from libtbx.path import makedirs_race
makedirs_race(path=pickle_dir)
n_caught = 0
for i_pair, pair in enumerate(hkl_cif.hkl_cif_pairs.values()):
cod_id = op.basename(pair[0])[:-4]
if i_pair % command_line.chunk.n != command_line.chunk.i:
continue
try:
cd = cod_data(cod_id=cod_id, hkl_cif_pair=pair)
except KeyboardInterrupt:
print "CAUGHT EXCEPTION: KeyboardInterrupt"
return
except Exception, e:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s: %s: %s" % (command_name, cod_id, str(e))
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
if cd.is_useful(co):
easy_pickle.dump(
file_name="%s/%s.pickle" % (pickle_dir, cod_id), obj=(cd.c_obs, cd.xray_structure, cd.edge_list)
)
print >> open("%s/qi_%s" % (pickle_dir, cod_id), "w"), cd.quick_info()
else:
print "filtering out:", cod_id
print "done_with:", cod_id
print
def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) +
" [options] directory|file...").enable_chunk(
easy_all=True).enable_multiprocessing()).process(args=args,
min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print "TIME BEGIN cod_refine:", date_and_time()
print
#
master_phil = get_master_phil()
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print
params = work_phil.extract()
#
qi_dict = {}
all_pickles = []
for arg in remaining_args:
if (op.isdir(arg)):
for node in sorted(os.listdir(arg)):
if (node.endswith(".pickle")):
all_pickles.append(op.join(arg, node))
elif (node.startswith("qi_") and len(node) == 10):
qi = open(op.join(arg, node)).read().splitlines()
if (len(qi) == 1):
cod_id = node[3:]
quick_info = eval(qi[0])
assert cod_id not in qi_dict
qi_dict[cod_id] = quick_info
elif (op.isfile(arg)):
all_pickles.append(arg)
else:
raise RuntimeError("Not a file or directory: %s" % arg)
print "Number of pickle files:", len(all_pickles)
print "Number of quick_infos:", len(qi_dict)
sort_choice = params.sorting_of_pickle_files
if (len(qi_dict) != 0 and sort_choice is not None):
print "Sorting pickle files by n_atoms * n_refl:", sort_choice
assert sort_choice in ["down", "up"]
def sort_pickle_files():
if (sort_choice == "down"): i_sign = -1
else: i_sign = 1
buffer = []
for i, path in enumerate(all_pickles):
cod_id = op.basename(path).split(".", 1)[0]
qi = qi_dict.get(cod_id)
if (qi is None): nn = 2**31
else: nn = qi[0] * qi[1] * qi[2]
buffer.append((nn, i_sign * i, path))
buffer.sort()
if (i_sign < 0):
buffer.reverse()
result = []
for elem in buffer:
result.append(elem[-1])
return result
all_pickles = sort_pickle_files()
print
#
rss = params.random_subset.size
if (rss is not None and rss > 0):
seed = params.random_subset.seed
print "Selecting subset of %d pickle files using random seed %d" % (
rss, seed)
mt = flex.mersenne_twister(seed=seed)
perm = mt.random_permutation(size=len(all_pickles))[:rss]
flags = flex.bool(len(all_pickles), False).set_selected(perm, True)
all_pickles = flex.select(all_pickles, permutation=flags.iselection())
print
#
from libtbx.path import makedirs_race
if (params.wdir_root is not None):
makedirs_race(path=params.wdir_root)
if (params.pickle_refined_dir is not None):
makedirs_race(path=params.pickle_refined_dir)
#
n_caught = 0
for i_pickle, pickle_file_name in enumerate(all_pickles):
if (i_pickle % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, pickle_file_name)
except KeyboardInterrupt:
print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt"
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s" % pickle_file_name
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
print "done_with: %s (%.2f seconds)" % (pickle_file_name,
tm.elapsed())
print
sys.stdout.flush()
print
print "Number of exceptions caught:", n_caught
#
show_times()
print
print "TIME END cod_refine:", date_and_time()
def run(args, command_call, command_line_add_options=None):
pdb_mirror_pdb = os.environ.get("PDB_MIRROR_PDB")
assert pdb_mirror_pdb is None or op.isdir(pdb_mirror_pdb)
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_line = (iotbx_option_parser(
usage=" ".join(command_call) +
" [options] file|directory...").enable_chunk(easy_all=True).
enable_multiprocessing().call_with_self_as_first_argument(
callable=command_line_add_options)).process(args=args)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return command_line, null_generator()
#
from libtbx.str_utils import show_string
ca = command_line.args
if (len(ca) == 0 and pdb_mirror_pdb is not None):
ca = [pdb_mirror_pdb]
file_infos = []
for arg in ca:
if (op.isfile(arg)):
bn = op.basename(arg)
if (bn.startswith("pdb_codes_")):
assert pdb_mirror_pdb is not None
for i_line, line in enumerate(open(arg).read().splitlines()):
flds = line.split(None, 1)
if (len(flds) == 0):
raise RuntimeError(
"Error interpreting pdb_codes file:\n"
" %s"
" line number: %d"
" line: %s" %
(show_string(arg), i_line + 1, show_string(line)))
pdb_code = flds[0]
atom_selection_string = None
if (len(flds) > 1):
atom_selection_string = flds[1]
file_name = op.join(pdb_mirror_pdb, pdb_code[1:3],
"pdb%s.ent.gz" % pdb_code)
file_infos.append(
file_info(name=file_name,
atom_selection_string=atom_selection_string))
elif (bn.startswith("file_names_")):
for file_name in open(arg).read().splitlines():
file_infos.append(file_info(name=file_name))
else:
file_infos.append(file_info(name=arg))
elif (op.isdir(arg)):
file_name_index = op.join(arg, "INDEX")
if (op.isfile(file_name_index)):
for relative_path in open(file_name_index).read().splitlines():
file_name = op.join(arg, relative_path)
file_infos.append(file_info(name=file_name))
else:
prev_len_file_infos = len(file_infos)
for relative_path in os.listdir(arg):
if (relative_path.endswith((".ent", ".ent.gz", "ent.Z",
".pdb", ".pdb.gz", "pdb.Z"))):
file_name = op.join(arg, relative_path)
file_infos.append(file_info(name=file_name))
if (len(file_infos) == prev_len_file_infos):
raise RuntimeError(
"No INDEX file and no pdb files found in directory: %s"
% show_string(arg))
else:
raise RuntimeError("Not a file or directory: %s" %
show_string(arg))
#
return command_line, pdb_inp_generator(file_infos=file_infos,
chunk_n=command_line.chunk.n,
chunk_i=command_line.chunk.i)
def run(args, command_name="phenix.tls"):
if (len(args) == 0): args = ["--help"]
usage_fmt = "%s pdb_file [parameters: file or command line string]"
des_fmt = "Example: %s model.pdb fit_tls_to.selection='%s' fit_tls_to.selection='%s'"
command_line = (iotbx_option_parser(
usage=usage_fmt % command_name, description=banner).option(
"--show_defaults",
action="store_true",
help="Do not output to the screen (except errors).").option(
"--silent",
action="store_true",
help="Suppress output to the screen.")).process(args=args)
#
log = sys.stdout
if (not command_line.options.silent):
utils.print_header("TLS tools", out=log)
if (command_line.options.show_defaults):
master_params.show(out=log)
print(file=log)
return
if (not command_line.options.silent):
print(banner, file=log)
#
processed_args = utils.process_command_line_args(
args=command_line.args, master_params=master_params, log=log)
reflection_files = processed_args.reflection_files
if (processed_args.crystal_symmetry is None):
raise Sorry("No crystal symmetry found.")
if (len(processed_args.pdb_file_names) == 0):
raise Sorry("No PDB file found.")
params = processed_args.params
if (not command_line.options.silent):
utils.print_header("Input parameters", out=log)
params.show(out=log)
params = params.extract()
#
if (processed_args.crystal_symmetry.unit_cell() is None
or processed_args.crystal_symmetry.space_group() is None):
raise Sorry("No CRYST1 record found.")
pdb_combined = iotbx.pdb.combine_unique_pdb_files(
file_names=processed_args.pdb_file_names)
pdb_combined.report_non_unique(out=log)
if (len(pdb_combined.unique_file_names) == 0):
raise Sorry("No coordinate file given.")
raw_records = pdb_combined.raw_records
try:
pdb_inp = iotbx.pdb.input(source_info=None,
lines=flex.std_string(raw_records))
except ValueError as e:
raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e))
model = mmtbx.model.manager(
model_input=pdb_inp,
restraint_objects=processed_args.cif_objects,
crystal_symmetry=processed_args.crystal_symmetry,
log=log)
if (not command_line.options.silent):
utils.print_header("TLS groups from PDB file header", out=log)
pdb_inp_tls = mmtbx.tls.tools.tls_from_pdb_inp(
remark_3_records=model._model_input.extract_remark_iii_records(3),
pdb_hierarchy=model.get_hierarchy())
#
tls_groups = []
if (pdb_inp_tls.tls_present):
if (pdb_inp_tls.error_string is not None):
raise Sorry(pdb_inp_tls.error_string)
pdb_tls = mmtbx.tls.tools.extract_tls_from_pdb(pdb_inp_tls=pdb_inp_tls,
model=model)
tls_groups = pdb_tls.pdb_inp_tls.tls_params
#
tls_selections_strings = []
#
if (len(tls_groups) == 0 and not command_line.options.silent):
print("No TLS groups found in PDB file header.", file=log)
else:
for i_seq, tls_group in enumerate(tls_groups):
tls_selections_strings.append(tls_group.selection_string)
if (not command_line.options.silent):
print("TLS group %d: %s" %
(i_seq + 1, tls_group.selection_string),
file=log)
mmtbx.tls.tools.show_tls_one_group(tlso=tls_group, out=log)
print(file=log)
#
if (len(tls_selections_strings) > 0 and len(params.selection) > 0):
raise Sorry(
"Two TLS selection sources found: PDB file header and parameters.")
if (len(params.selection) > 0):
tls_selections_strings = params.selection
if ([params.combine_tls, params.extract_tls].count(True) > 1):
raise Sorry(
"Cannot simultaneously pereform: combine_tls and extract_tls")
if ([params.combine_tls, params.extract_tls].count(True) > 0):
if (len(tls_selections_strings) == 0):
raise Sorry("No TLS selections found.")
#
if (len(tls_selections_strings)):
if (not command_line.options.silent):
utils.print_header("TLS groups selections", out=log)
selections = utils.get_atom_selections(
model=model, selection_strings=tls_selections_strings)
if (not command_line.options.silent):
print("Number of TLS groups: ", len(selections), file=log)
print("Number of atoms: %d" % model.get_number_of_atoms(),
file=log)
n_atoms_in_tls = 0
for sel_a in selections:
n_atoms_in_tls += sel_a.size()
if (not command_line.options.silent):
print("Number of atoms in TLS groups: %d" % n_atoms_in_tls,
file=log)
print(file=log)
assert len(tls_selections_strings) == len(selections)
if (not command_line.options.silent):
for sel_a, sel_s in zip(selections, tls_selections_strings):
print("Selection string:\n%s" % sel_s, file=log)
print("selects %d atoms." % sel_a.size(), file=log)
print(file=log)
print("Ready-to-use in phenix.refine:\n", file=log)
for sel_a, sel_s in zip(selections, tls_selections_strings):
print(sel_s, file=log)
#
ofn = params.output_file_name
if (ofn is None):
ofn = os.path.splitext(
os.path.basename(processed_args.pdb_file_names[0]))[0]
if (len(processed_args.pdb_file_names) > 1):
ofn = ofn + "_el_al"
if (params.combine_tls):
ofn = ofn + "_combine_tls.pdb"
elif (params.extract_tls):
ofn = ofn + "_extract_tls.pdb"
else:
ofn = None
if (ofn is not None):
ofo = open(ofn, "w")
#
if (params.extract_tls):
utils.print_header(
"Fit TLS matrices to B-factors of selected sets of atoms", out=log)
tlsos = mmtbx.tls.tools.generate_tlsos(
selections=selections,
xray_structure=model.get_xray_structure(),
value=0.0)
for rt, rl, rs in [[1, 0, 1], [1, 1, 1], [0, 1, 1], [1, 0, 0],
[0, 1, 0], [0, 0, 1], [1, 1, 1], [0, 0, 1]] * 10:
tlsos = mmtbx.tls.tools.tls_from_uanisos(
xray_structure=model.get_xray_structure(),
selections=selections,
tlsos_initial=tlsos,
number_of_macro_cycles=10,
max_iterations=100,
refine_T=rt,
refine_L=rl,
refine_S=rs,
enforce_positive_definite_TL=params.
enforce_positive_definite_TL,
verbose=-1,
out=log)
mmtbx.tls.tools.show_tls(tlsos=tlsos, out=log)
u_cart_from_tls = mmtbx.tls.tools.u_cart_from_tls(
sites_cart=model.get_sites_cart(),
selections=selections,
tlsos=tlsos)
unit_cell = model.get_xray_structure().unit_cell()
for i_seq, sc in enumerate(model.get_xray_structure().scatterers()):
if (u_cart_from_tls[i_seq] != (0, 0, 0, 0, 0, 0)):
u_star_tls = adptbx.u_cart_as_u_star(
unit_cell, tuple(u_cart_from_tls[i_seq]))
sc.u_star = tuple(
flex.double(sc.u_star) - flex.double(u_star_tls))
for sel in selections:
model.get_xray_structure().convert_to_isotropic(selection=sel)
mmtbx.tls.tools.remark_3_tls(tlsos=tlsos,
selection_strings=tls_selections_strings,
out=ofo)
#
if (params.combine_tls):
utils.print_header("Combine B_tls with B_residual", out=log)
mmtbx.tls.tools.combine_tls_and_u_local(
xray_structure=model.get_xray_structure(),
tls_selections=selections,
tls_groups=tls_groups)
print("All done.", file=log)
#
if (ofn is not None):
utils.print_header("Write output PDB file %s" % ofn, out=log)
model.set_sites_cart_from_xrs()
pdb_str = model.model_as_pdb()
ofo.write(pdb_str)
ofo.close()
print("All done.", file=log)
def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) + " [options] directory|file...")
.enable_chunk(easy_all=True)
.enable_multiprocessing()
).process(args=args, min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print "TIME BEGIN pdb_dev:", date_and_time()
print
libtbx.utils.host_and_user().show()
print
sys.stdout.flush()
#
from cctbx.omz import cod_refine
master_phil = cod_refine.get_master_phil(
max_atoms=None,
f_calc_options_algorithm="direct *fft",
bulk_solvent_correction=True)
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print
params = work_phil.extract()
#
mtz_pdb_pairs = []
arg_iter = iter(remaining_args)
pdb_v3_mirror_dir = os.environ.get("PDB_MIRROR_PDB")
assert pdb_v3_mirror_dir is None or op.isdir(pdb_v3_mirror_dir)
cci_pdbmtz_path = os.environ.get("CCI_PDBMTZ")
assert cci_pdbmtz_path is None or op.isdir(cci_pdbmtz_path)
for arg in arg_iter:
def get_next(expected_exts):
def raise_bad_file(what, fn=None):
msg = "%s file name (%s expected)" % (what, " or ".join(expected_exts))
if (fn is None):
msg += "."
else:
msg += ": " + show_string(fn)
raise RuntimeError(msg)
try:
arg = arg_iter.next()
except StopIteration:
raise_bad_file("Missing")
if (not arg.endswith(tuple(expected_exts))):
raise_bad_file("Unexpected", arg)
return arg
if (op.isfile(arg) and arg.endswith((".mtz", ".pdb", ".ent"))):
if (arg.endswith(".mtz")):
fn_mtz = arg
fn_pdb = get_next([".pdb", ".ent"])
else:
fn_pdb = arg
fn_mtz = get_next([".mtz"])
else:
fn_mtz = arg+".mtz"
def raise_mtz_but_no_pdb():
raise RuntimeError(
"MTZ file found but no PDB file: %s" % show_string(fn_mtz))
if (op.isfile(fn_mtz)):
for ext in [".pdb", ".ent"]:
fn_pdb = arg+ext
if (op.isfile(fn_pdb)):
break
else:
raise_mtz_but_no_pdb()
else:
fn_mtz = op.join(cci_pdbmtz_path, arg+".mtz")
if (not op.isfile(fn_mtz)):
raise RuntimeError(
"MTZ file not found: %s" % show_string(fn_mtz))
fn_pdb = op.join(pdb_v3_mirror_dir, arg[1:3], "pdb"+arg+".ent.gz")
if (not op.isfile(fn_pdb)):
raise_mtz_but_no_pdb()
mtz_pdb_pairs.append((fn_mtz, fn_pdb))
#
n_caught = 0
for i_pair,mtz_pdb_pair in enumerate(mtz_pdb_pairs):
if (i_pair % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, mtz_pdb_pair)
except KeyboardInterrupt:
print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt"
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s" % ", ".join(mtz_pdb_pair)
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
print "done_with: %s, %s (%.2f seconds)" % (
mtz_pdb_pair + (tm.elapsed(),))
print
sys.stdout.flush()
print
print "Number of exceptions caught:", n_caught
#
show_times()
print
print "TIME END pdb_dev:", date_and_time()
sys.stdout.flush()
def run(args, command_name = "phenix.tls"):
if(len(args) == 0): args = ["--help"]
usage_fmt = "%s pdb_file [parameters: file or command line string]"
des_fmt = "Example: %s model.pdb fit_tls_to.selection='%s' fit_tls_to.selection='%s'"
command_line = (iotbx_option_parser(
usage = usage_fmt % command_name,
description = banner)
.option("--show_defaults",
action="store_true",
help="Do not output to the screen (except errors).")
.option("--silent",
action="store_true",
help="Suppress output to the screen.")
).process(args=args)
#
log = sys.stdout
if(not command_line.options.silent):
utils.print_header("TLS tools", out = log)
if(command_line.options.show_defaults):
master_params.show(out = log)
print >> log
return
if(not command_line.options.silent):
print >> log, banner
#
processed_args = utils.process_command_line_args(args = command_line.args,
master_params = master_params, log = log)
reflection_files = processed_args.reflection_files
if(processed_args.crystal_symmetry is None):
raise Sorry("No crystal symmetry found.")
if(len(processed_args.pdb_file_names) == 0):
raise Sorry("No PDB file found.")
params = processed_args.params
if(not command_line.options.silent):
utils.print_header("Input parameters", out = log)
params.show(out = log)
params = params.extract()
#
if(processed_args.crystal_symmetry.unit_cell() is None or
processed_args.crystal_symmetry.space_group() is None):
raise Sorry("No CRYST1 record found.")
mmtbx_pdb_file = utils.pdb_file(
pdb_file_names = processed_args.pdb_file_names,
cif_objects = processed_args.cif_objects,
crystal_symmetry = processed_args.crystal_symmetry,
log = log)
#
if(not command_line.options.silent):
utils.print_header("TLS groups from PDB file header", out = log)
pdb_inp_tls = mmtbx.tls.tools.tls_from_pdb_inp(
remark_3_records = mmtbx_pdb_file.pdb_inp.extract_remark_iii_records(3),
pdb_hierarchy = mmtbx_pdb_file.pdb_inp.construct_hierarchy())
#
tls_groups = []
if(pdb_inp_tls.tls_present):
if(pdb_inp_tls.error_string is not None):
raise Sorry(pdb_inp_tls.error_string)
mmtbx_pdb_file.set_ppf()
xray_structure = get_xrs_helper(mmtbx_pdb_file = mmtbx_pdb_file, log = log,
silent = command_line.options.silent)
pdb_tls = mmtbx.tls.tools.extract_tls_from_pdb(
pdb_inp_tls = pdb_inp_tls,
all_chain_proxies = mmtbx_pdb_file.processed_pdb_file.all_chain_proxies,
xray_structure = xray_structure)
tls_groups = pdb_tls.pdb_inp_tls.tls_params
#
tls_selections_strings = []
#
if(len(tls_groups) == 0 and not command_line.options.silent):
print >> log, "No TLS groups found in PDB file header."
else:
for i_seq, tls_group in enumerate(tls_groups):
tls_selections_strings.append(tls_group.selection_string)
if(not command_line.options.silent):
print >> log, "TLS group %d: %s" % (i_seq+1, tls_group.selection_string)
mmtbx.tls.tools.show_tls_one_group(tlso = tls_group, out = log)
print >> log
#
if(len(tls_selections_strings) > 0 and len(params.selection) > 0):
raise Sorry("Two TLS selection sources found: PDB file header and parameters.")
if(len(params.selection) > 0):
tls_selections_strings = params.selection
xray_structure = get_xrs_helper(mmtbx_pdb_file = mmtbx_pdb_file, log = log,
silent = command_line.options.silent)
if([params.combine_tls, params.extract_tls].count(True) > 1):
raise Sorry("Cannot simultaneously pereform: combine_tls and extract_tls")
if([params.combine_tls, params.extract_tls].count(True) > 0):
if(len(tls_selections_strings)==0):
raise Sorry("No TLS selections found.")
#
if(len(tls_selections_strings)):
if(not command_line.options.silent):
utils.print_header("TLS groups selections", out = log)
selections = utils.get_atom_selections(
all_chain_proxies = mmtbx_pdb_file.processed_pdb_file.all_chain_proxies,
selection_strings = tls_selections_strings,
xray_structure = xray_structure)
if(not command_line.options.silent):
print >> log, "Number of TLS groups: ", len(selections)
print >> log, "Number of atoms: %d" % xray_structure.scatterers().size()
n_atoms_in_tls = 0
for sel_a in selections:
n_atoms_in_tls += sel_a.size()
if(not command_line.options.silent):
print >> log, "Number of atoms in TLS groups: %d" % n_atoms_in_tls
print >> log
assert len(tls_selections_strings) == len(selections)
if(not command_line.options.silent):
for sel_a, sel_s in zip(selections,tls_selections_strings):
print >> log, "Selection string:\n%s" % sel_s
print >> log, "selects %d atoms." % sel_a.size()
print >> log
print >> log, "Ready-to-use in phenix.refine:\n"
for sel_a, sel_s in zip(selections,tls_selections_strings):
print >> log, sel_s
#
ofn = params.output_file_name
if(ofn is None):
ofn = os.path.splitext(os.path.basename(processed_args.pdb_file_names[0]))[0]
if(len(processed_args.pdb_file_names) > 1):
ofn = ofn+"_el_al"
if(params.combine_tls):
ofn = ofn+"_combine_tls.pdb"
elif(params.extract_tls):
ofn = ofn+"_extract_tls.pdb"
else: ofn = None
if(ofn is not None):
ofo = open(ofn, "w")
#
if(params.extract_tls):
utils.print_header(
"Fit TLS matrices to B-factors of selected sets of atoms", out = log)
tlsos = mmtbx.tls.tools.generate_tlsos(
selections = selections,
xray_structure = xray_structure,
value = 0.0)
for rt,rl,rs in [[1,0,1],[1,1,1],[0,1,1],
[1,0,0],[0,1,0],[0,0,1],[1,1,1],
[0,0,1]]*10:
tlsos = mmtbx.tls.tools.tls_from_uanisos(
xray_structure = xray_structure,
selections = selections,
tlsos_initial = tlsos,
number_of_macro_cycles = 10,
max_iterations = 100,
refine_T = rt,
refine_L = rl,
refine_S = rs,
enforce_positive_definite_TL = params.enforce_positive_definite_TL,
verbose = -1,
out = log)
mmtbx.tls.tools.show_tls(tlsos = tlsos, out = log)
u_cart_from_tls = mmtbx.tls.tools.u_cart_from_tls(
sites_cart = xray_structure.sites_cart(),
selections = selections,
tlsos = tlsos)
unit_cell = xray_structure.unit_cell()
for i_seq, sc in enumerate(xray_structure.scatterers()):
if(u_cart_from_tls[i_seq] != (0,0,0,0,0,0)):
u_star_tls = adptbx.u_cart_as_u_star(unit_cell,
tuple(u_cart_from_tls[i_seq]))
sc.u_star = tuple(flex.double(sc.u_star) - flex.double(u_star_tls))
for sel in selections:
xray_structure.convert_to_isotropic(selection = sel)
mmtbx.tls.tools.remark_3_tls(tlsos = tlsos,
selection_strings = tls_selections_strings, out = ofo)
#
if(params.combine_tls):
utils.print_header("Combine B_tls with B_residual", out = log)
mmtbx.tls.tools.combine_tls_and_u_local(xray_structure = xray_structure,
tls_selections = selections, tls_groups = tls_groups)
print >> log, "All done."
#
if(ofn is not None):
utils.print_header("Write output PDB file %s"%ofn, out = log)
utils.write_pdb_file(
xray_structure = xray_structure,
pdb_hierarchy = mmtbx_pdb_file.processed_pdb_file.all_chain_proxies.pdb_hierarchy,
out = ofo)
ofo.close()
print >> log, "All done."
def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) +
" [options] directory|file...").enable_chunk(
easy_all=True).enable_multiprocessing()).process(args=args,
min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print("TIME BEGIN pdb_dev:", date_and_time())
print()
libtbx.utils.host_and_user().show()
print()
sys.stdout.flush()
#
from cctbx.omz import cod_refine
master_phil = cod_refine.get_master_phil(
max_atoms=None,
f_calc_options_algorithm="direct *fft",
bulk_solvent_correction=True)
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print()
params = work_phil.extract()
#
mtz_pdb_pairs = []
arg_iter = iter(remaining_args)
pdb_v3_mirror_dir = os.environ.get("PDB_MIRROR_PDB")
assert pdb_v3_mirror_dir is None or op.isdir(pdb_v3_mirror_dir)
cci_pdbmtz_path = os.environ.get("CCI_PDBMTZ")
assert cci_pdbmtz_path is None or op.isdir(cci_pdbmtz_path)
for arg in arg_iter:
def get_next(expected_exts):
def raise_bad_file(what, fn=None):
msg = "%s file name (%s expected)" % (
what, " or ".join(expected_exts))
if (fn is None):
msg += "."
else:
msg += ": " + show_string(fn)
raise RuntimeError(msg)
try:
arg = next(arg_iter)
except StopIteration:
raise_bad_file("Missing")
if (not arg.endswith(tuple(expected_exts))):
raise_bad_file("Unexpected", arg)
return arg
if (op.isfile(arg) and arg.endswith((".mtz", ".pdb", ".ent"))):
if (arg.endswith(".mtz")):
fn_mtz = arg
fn_pdb = get_next([".pdb", ".ent"])
else:
fn_pdb = arg
fn_mtz = get_next([".mtz"])
else:
fn_mtz = arg + ".mtz"
def raise_mtz_but_no_pdb():
raise RuntimeError("MTZ file found but no PDB file: %s" %
show_string(fn_mtz))
if (op.isfile(fn_mtz)):
for ext in [".pdb", ".ent"]:
fn_pdb = arg + ext
if (op.isfile(fn_pdb)):
break
else:
raise_mtz_but_no_pdb()
else:
fn_mtz = op.join(cci_pdbmtz_path, arg + ".mtz")
if (not op.isfile(fn_mtz)):
raise RuntimeError("MTZ file not found: %s" %
show_string(fn_mtz))
fn_pdb = op.join(pdb_v3_mirror_dir, arg[1:3],
"pdb" + arg + ".ent.gz")
if (not op.isfile(fn_pdb)):
raise_mtz_but_no_pdb()
mtz_pdb_pairs.append((fn_mtz, fn_pdb))
#
n_caught = 0
for i_pair, mtz_pdb_pair in enumerate(mtz_pdb_pairs):
if (i_pair % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, mtz_pdb_pair)
except KeyboardInterrupt:
print("CAUGHT EXCEPTION: KeyboardInterrupt", file=sys.stderr)
traceback.print_exc()
print(file=sys.stderr)
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print("CAUGHT EXCEPTION: %s" % ", ".join(mtz_pdb_pair),
file=sys.stderr)
traceback.print_exc()
print(file=sys.stderr)
sys.stderr.flush()
n_caught += 1
else:
print("done_with: %s, %s (%.2f seconds)" % (mtz_pdb_pair +
(tm.elapsed(), )))
print()
sys.stdout.flush()
print()
print("Number of exceptions caught:", n_caught)
#
show_times()
print()
print("TIME END pdb_dev:", date_and_time())
sys.stdout.flush()
def run(self, args, command_name, out=sys.stdout):
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description='Example: %s data.mtz data.mtz ref_model.pdb' %
command_name).option(
None,
"--show_defaults",
action="store_true",
help="Show list of parameters.")).process(args=args)
cif_file = None
processed_args = utils.process_command_line_args(
args=args, log=sys.stdout, master_params=master_phil)
params = processed_args.params
if (params is None): params = master_phil
self.params = params.extract().ensemble_probability
pdb_file_names = processed_args.pdb_file_names
if len(pdb_file_names) != 1:
raise Sorry("Only one PDB structure may be used")
pdb_file = file_reader.any_file(pdb_file_names[0])
self.log = multi_out()
self.log.register(label="stdout", file_object=sys.stdout)
self.log.register(label="log_buffer",
file_object=StringIO(),
atexit_send_to=None)
sys.stderr = self.log
log_file = open(
pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.log', "w")
self.log.replace_stringio(old_label="log_buffer",
new_label="log",
new_file_object=log_file)
utils.print_header(command_name, out=self.log)
params.show(out=self.log)
#
f_obs = None
r_free_flags = None
reflection_files = processed_args.reflection_files
if self.params.fobs_vs_fcalc_post_nll:
if len(reflection_files) == 0:
raise Sorry(
"Fobs from input MTZ required for fobs_vs_fcalc_post_nll")
if len(reflection_files) > 0:
crystal_symmetry = processed_args.crystal_symmetry
print('Reflection file : ',
processed_args.reflection_file_names[0],
file=self.log)
utils.print_header("Model and data statistics", out=self.log)
rfs = reflection_file_server(
crystal_symmetry=crystal_symmetry,
reflection_files=processed_args.reflection_files,
log=self.log)
parameters = extract_xtal_data.data_and_flags_master_params(
).extract()
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server=rfs,
parameters=parameters,
data_parameter_scope="refinement.input.xray_data",
flags_parameter_scope="refinement.input.xray_data.r_free_flags",
data_description="X-ray data",
keep_going=True,
log=self.log)
f_obs = determine_data_and_flags_result.f_obs
number_of_reflections = f_obs.indices().size()
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(
data=flex.bool(f_obs.data().size(), False))
# process PDB
pdb_file.assert_file_type("pdb")
#
pdb_in = hierarchy.input(file_name=pdb_file.file_name)
ens_pdb_hierarchy = pdb_in.construct_hierarchy()
ens_pdb_hierarchy.atoms().reset_i_seq()
ens_pdb_xrs_s = pdb_in.input.xray_structures_simple()
number_structures = len(ens_pdb_xrs_s)
print('Number of structure in ensemble : ',
number_structures,
file=self.log)
# Calculate sigmas from input map only
if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None:
# process MTZ
input_file = file_reader.any_file(
self.params.ensemble_sigma_map_input)
if input_file.file_type == "hkl":
if input_file.file_object.file_type() != "ccp4_mtz":
raise Sorry("Only MTZ format accepted for map input")
else:
mtz_file = input_file
else:
raise Sorry("Only MTZ format accepted for map input")
miller_arrays = mtz_file.file_server.miller_arrays
map_coeffs_1 = miller_arrays[0]
#
xrs_list = []
for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
# get sigma levels from ensemble fc for each structure
xrs = get_map_sigma(ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
map_coeffs_1=map_coeffs_1,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
log=self.log)
xrs_list.append(xrs)
# write ensemble pdb file, occupancies as sigma level
filename = pdb_file_names[0].split('/')[-1].replace(
'.pdb',
'') + '_vs_' + self.params.ensemble_sigma_map_input.replace(
'.mtz', '') + '_pensemble.pdb'
write_ensemble_pdb(filename=filename,
xrs_list=xrs_list,
ens_pdb_hierarchy=ens_pdb_hierarchy)
# Do full analysis vs Fobs
else:
model_map_coeffs = []
fmodel = None
# Get <fcalc>
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
ens_pdb_xrs.set_occupancies(1.0)
if model == 0:
# If mtz not supplied get fobs from xray structure...
# Use input Fobs for scoring against nll
if self.params.fobs_vs_fcalc_post_nll:
dummy_fobs = f_obs
else:
if f_obs == None:
if self.params.fcalc_high_resolution == None:
raise Sorry(
"Please supply high resolution limit or input mtz file."
)
dummy_dmin = self.params.fcalc_high_resolution
dummy_dmax = self.params.fcalc_low_resolution
else:
print(
'Supplied mtz used to determine high and low resolution cuttoffs',
file=self.log)
dummy_dmax, dummy_dmin = f_obs.d_max_min()
#
dummy_fobs = abs(
ens_pdb_xrs.structure_factors(
d_min=dummy_dmin).f_calc())
dummy_fobs.set_observation_type_xray_amplitude()
# If mtz supplied, free flags are over written to prevent array size error
r_free_flags = dummy_fobs.array(
data=flex.bool(dummy_fobs.data().size(), False))
#
fmodel = utils.fmodel_simple(
scattering_table="wk1995",
xray_structures=[ens_pdb_xrs],
f_obs=dummy_fobs,
target_name='ls',
bulk_solvent_and_scaling=False,
r_free_flags=r_free_flags)
f_calc_ave = fmodel.f_calc().array(
data=fmodel.f_calc().data() * 0).deep_copy()
# XXX Important to ensure scale is identical for each model and <model>
fmodel.set_scale_switch = 1.0
f_calc_ave_total = fmodel.f_calc().data().deep_copy()
else:
fmodel.update_xray_structure(xray_structure=ens_pdb_xrs,
update_f_calc=True,
update_f_mask=False)
f_calc_ave_total += fmodel.f_calc().data().deep_copy()
print('Model :', model + 1, file=self.log)
print("\nStructure vs real Fobs (no bulk solvent or scaling)",
file=self.log)
print('Rwork : %5.4f ' % fmodel.r_work(),
file=self.log)
print('Rfree : %5.4f ' % fmodel.r_free(),
file=self.log)
print('K1 : %5.4f ' % fmodel.scale_k1(),
file=self.log)
fcalc_edm = fmodel.electron_density_map()
fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type='Fc')
fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset(
column_root_label='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_mtz_dataset.mtz_object().write(
file_name=str(model + 1) + "_Fc.mtz")
model_map_coeffs.append(fcalc_map_coeffs.deep_copy())
fmodel.update(f_calc=f_calc_ave.array(f_calc_ave_total /
number_structures))
print("\nEnsemble vs real Fobs (no bulk solvent or scaling)",
file=self.log)
print('Rwork : %5.4f ' % fmodel.r_work(), file=self.log)
print('Rfree : %5.4f ' % fmodel.r_free(), file=self.log)
print('K1 : %5.4f ' % fmodel.scale_k1(), file=self.log)
# Get <Fcalc> map
fcalc_ave_edm = fmodel.electron_density_map()
fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients(
map_type='Fc').deep_copy()
fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset(
column_root_label='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_ave_mtz_dataset.mtz_object().write(file_name="aveFc.mtz")
fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map()
fcalc_ave_map_coeffs.apply_volume_scaling()
fcalc_ave_map_data = fcalc_ave_map_coeffs.real_map_unpadded()
fcalc_ave_map_stats = maptbx.statistics(fcalc_ave_map_data)
print("<Fcalc> Map Stats :", file=self.log)
fcalc_ave_map_stats.show_summary(f=self.log)
offset = fcalc_ave_map_stats.min()
model_neg_ll = []
number_previous_scatters = 0
# Run through structure list again and get probability
xrs_list = []
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
if self.params.verbose:
print('\n\nModel : ',
model + 1,
file=self.log)
# Get model atom sigmas vs Fcalc
fcalc_map = model_map_coeffs[model].fft_map()
fcalc_map.apply_volume_scaling()
fcalc_map_data = fcalc_map.real_map_unpadded()
fcalc_map_stats = maptbx.statistics(fcalc_map_data)
if self.params.verbose:
print("Fcalc map stats :", file=self.log)
fcalc_map_stats.show_summary(f=self.log)
xrs = get_map_sigma(
ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
fft_map_1=fcalc_map,
model_i=model,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
number_previous_scatters=number_previous_scatters,
log=self.log)
fcalc_sigmas = xrs.scatterers().extract_occupancies()
del fcalc_map
# Get model atom sigmas vs <Fcalc>
xrs = get_map_sigma(
ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
fft_map_1=fcalc_ave_map_coeffs,
model_i=model,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
number_previous_scatters=number_previous_scatters,
log=self.log)
### For testing other residue averaging options
#print xrs.residue_selections
fcalc_ave_sigmas = xrs.scatterers().extract_occupancies()
# Probability of model given <model>
prob = fcalc_ave_sigmas / fcalc_sigmas
# XXX debug option
if False:
for n, p in enumerate(prob):
print(' {0:5d} {1:5.3f}'.format(n, p), file=self.log)
# Set probabilty between 0 and 1
# XXX Make Histogram / more stats
prob_lss_zero = flex.bool(prob <= 0)
prob_grt_one = flex.bool(prob > 1)
prob.set_selected(prob_lss_zero, 0.001)
prob.set_selected(prob_grt_one, 1.0)
xrs.set_occupancies(prob)
xrs_list.append(xrs)
sum_neg_ll = sum(-flex.log(prob))
model_neg_ll.append((sum_neg_ll, model))
if self.params.verbose:
print('Model probability stats :', file=self.log)
print(prob.min_max_mean().show(), file=self.log)
print(' Count < 0.0 : ',
prob_lss_zero.count(True),
file=self.log)
print(' Count > 1.0 : ',
prob_grt_one.count(True),
file=self.log)
# For averaging by residue
number_previous_scatters += ens_pdb_xrs.sites_cart().size()
# write ensemble pdb file, occupancies as sigma level
write_ensemble_pdb(
filename=pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.pdb',
xrs_list=xrs_list,
ens_pdb_hierarchy=ens_pdb_hierarchy)
# XXX Test ordering models by nll
# XXX Test removing nth percentile atoms
if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll:
for percentile in [1.0, 0.975, 0.95, 0.9, 0.8, 0.6, 0.2]:
model_neg_ll = sorted(model_neg_ll)
f_calc_ave_total_reordered = None
print_list = []
for i_neg_ll in model_neg_ll:
xrs = xrs_list[i_neg_ll[1]]
nll_occ = xrs.scatterers().extract_occupancies()
# Set q=0 nth percentile atoms
sorted_nll_occ = sorted(nll_occ, reverse=True)
number_atoms = len(sorted_nll_occ)
percentile_prob_cutoff = sorted_nll_occ[
int(number_atoms * percentile) - 1]
cutoff_selections = flex.bool(
nll_occ < percentile_prob_cutoff)
cutoff_nll_occ = flex.double(nll_occ.size(),
1.0).set_selected(
cutoff_selections,
0.0)
#XXX Debug
if False:
print('\nDebug')
for x in range(len(cutoff_selections)):
print(cutoff_selections[x], nll_occ[x],
cutoff_nll_occ[x])
print(percentile)
print(percentile_prob_cutoff)
print(cutoff_selections.count(True))
print(cutoff_selections.size())
print(cutoff_nll_occ.count(0.0))
print('Count q = 1 : ',
cutoff_nll_occ.count(1.0))
print('Count scatterers size : ',
cutoff_nll_occ.size())
xrs.set_occupancies(cutoff_nll_occ)
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True,
update_f_mask=True)
if f_calc_ave_total_reordered == None:
f_calc_ave_total_reordered = fmodel.f_calc().data(
).deep_copy()
f_mask_ave_total_reordered = fmodel.f_masks(
)[0].data().deep_copy()
cntr = 1
else:
f_calc_ave_total_reordered += fmodel.f_calc().data(
).deep_copy()
f_mask_ave_total_reordered += fmodel.f_masks(
)[0].data().deep_copy()
cntr += 1
fmodel.update(
f_calc=f_calc_ave.array(
f_calc_ave_total_reordered / cntr).deep_copy(),
f_mask=f_calc_ave.array(
f_mask_ave_total_reordered / cntr).deep_copy())
# Update solvent and scale
# XXX Will need to apply_back_trace on latest version
fmodel.set_scale_switch = 0
fmodel.update_all_scales()
# Reset occ for outout
xrs.set_occupancies(nll_occ)
# k1 updated vs Fobs
if self.params.fobs_vs_fcalc_post_nll:
print_list.append([
cntr, i_neg_ll[0], i_neg_ll[1],
fmodel.r_work(),
fmodel.r_free()
])
# Order models by nll and print summary
print(
'\nModels ranked by nll <Fcalc> R-factors recalculated',
file=self.log)
print('Percentile cutoff : {0:5.3f}'.format(percentile),
file=self.log)
xrs_list_sorted_nll = []
print(' | NLL <Rw> <Rf> Ens Model',
file=self.log)
for info in print_list:
print(' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'.
format(
info[0],
info[1],
info[3],
info[4],
info[2] + 1,
),
file=self.log)
xrs_list_sorted_nll.append(xrs_list[info[2]])
# Output nll ordered ensemble
write_ensemble_pdb(
filename='nll_ordered_' +
pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.pdb',
xrs_list=xrs_list_sorted_nll,
ens_pdb_hierarchy=ens_pdb_hierarchy)
def run(args, command_name):
from iotbx.option_parser import option_parser as iotbx_option_parser
from libtbx import easy_pickle
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_line = (iotbx_option_parser(
usage=command_name+" [options] [cod_id...]")
.enable_chunk(easy_all=True)
.enable_multiprocessing()
.option(None, "--max_atoms",
type="int",
default=99,
metavar="INT")
.option(None, "--min_distance",
type="float",
default=0.5,
metavar="FLOAT")
.option(None, "--i_obs_i_calc_plot",
action="store_true",
default=False)
.option(None, "--f_obs_f_calc_plot",
action="store_true",
default=False)
.option(None, "--max_fan_outlier_fraction",
type="float",
default=0.05,
metavar="FLOAT")
.option(None, "--fan_offset_low",
type="float",
default=0.05,
metavar="FLOAT")
.option(None, "--fan_offset_high",
type="float",
default=0.10,
metavar="FLOAT")
.option(None, "--min_f_obs_f_calc_correlation",
type="float",
default=0.99,
metavar="FLOAT")
.option(None, "--at_least_one_special_position",
action="store_true",
default=False)
.option(None, "--pickle_dir",
type="str",
default="cod_ma_xs",
metavar="PATH")
).process(args=args)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=[command_name])):
show_times()
return
co = command_line.options
#
from iotbx.cif import cod_tools
hkl_cif = cod_tools.build_hkl_cif(cod_ids=command_line.args)
hkl_cif.show_summary()
#
pickle_dir = co.pickle_dir
if (co.at_least_one_special_position):
pickle_dir += "_special"
if (not op.isdir(pickle_dir)):
from libtbx.path import makedirs_race
makedirs_race(path=pickle_dir)
n_caught = 0
for i_pair,pair in enumerate(hkl_cif.hkl_cif_pairs.values()):
cod_id = op.basename(pair[0])[:-4]
if (i_pair % command_line.chunk.n != command_line.chunk.i): continue
try:
cd = cod_data(cod_id=cod_id, hkl_cif_pair=pair)
except KeyboardInterrupt:
print("CAUGHT EXCEPTION: KeyboardInterrupt")
return
except Exception as e:
sys.stdout.flush()
print("CAUGHT EXCEPTION: %s: %s: %s" % (command_name, cod_id, str(e)), file=sys.stderr)
traceback.print_exc()
print(file=sys.stderr)
sys.stderr.flush()
n_caught += 1
else:
if (cd.is_useful(co)):
easy_pickle.dump(
file_name="%s/%s.pickle" % (pickle_dir, cod_id),
obj=(cd.c_obs, cd.xray_structure, cd.edge_list))
print(cd.quick_info(), file=open("%s/qi_%s" % (pickle_dir, cod_id), "w"))
else:
print("filtering out:", cod_id)
print("done_with:", cod_id)
print()
print()
print("Number of exceptions caught:", n_caught)
#
show_times()
