def run(args):
# create parser
#logger = multi_out() #logging.getLogger('main')
#logger.register('stdout', sys.stdout)
#logger = null_out()
logger = multi_out()
logger.register('stderr', sys.stderr)
logger2 = multi_out()
logger2.register('stdout', sys.stdout)
#only omegalyze output is sent to stdout for backward compatibility with
# MolProbity website
parser = CCTBXParser(program_class=omegalyze.Program, logger=logger)
namespace = parser.parse_args(sys.argv[1:])
# start program
print('Starting job', file=logger)
print('=' * 79, file=logger)
task = omegalyze.Program(parser.data_manager,
parser.working_phil.extract(),
logger=logger2)
# validate inputs
task.validate()
# run program
task.run()
# stop timer
print('', file=logger)
print('=' * 79, file=logger)
print('Job complete', file=logger)
show_total_time(out=logger)
def run(args):
"The main program, if run from CCTBX / PHENIX."
logger = multi_out()
logger.register('stderr', sys.stderr)
logger2 = multi_out()
logger2.register('stdout', sys.stdout)
logger3 = multi_out()
# TO DIAGNOSE OPTIONS TROUBLES:
# logger3.register('verbiage', open("suitename.stderr.log", "w"))
parser = dualparse.parseArgs(Program, logger3)
working_phil = parser.working_phil
options = working_phil.extract().suitename
# now we call into the core of suitename itself
if options.version:
print(version, file=logger2)
return
if options.infile == "" or options.infile == "-" or options.residuein or options.suitein:
# let the core figure out the input
main(optionsIn=options, outFile=logger2, errorFile=logger)
else:
type, ext = analyzeFileType(options.infile)
if type == "":
logger.write("File extension " + str(ext) + " not recognized\n")
return
if type == "pdb":
suites.main(options=options, outFile=logger2, errorFile=logger)
else:
# help the core figure out the input file type
if type == "kinemage":
options.suitein = True
elif type == "dangle":
options.residuein = True
main(optionsIn=options, outFile=logger2, errorFile=logger)
def run(args):
# create parser
logger = multi_out()
logger.register('stderr', sys.stderr)
logger2 = multi_out()
logger2.register('stdout', sys.stdout)
parser = CCTBXParser(program_class=cablam.Program, logger=logger)
namespace = parser.parse_args(sys.argv[1:])
# start program
print('Starting job', file=logger)
print('=' * 79, file=logger)
task = cablam.Program(parser.data_manager,
parser.working_phil.extract(),
logger=logger2)
# validate inputs
task.validate()
# run program
task.run()
# stop timer
print('', file=logger)
print('=' * 79, file=logger)
print('Job complete', file=logger)
show_total_time(out=logger)
def read_models(self):
# Read Nat's reference model from an MTZ file. XXX The observation
# type is given as F, not I--should they be squared? Check with Nat!
log = open("%s.log" % self.params.output.prefix, "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
print("I model", file=out)
if self.params.model is not None:
from xfel.merging.general_fcalc import run as run_fmodel
i_model = run_fmodel(self.params)
self.params.target_unit_cell = i_model.unit_cell()
self.params.target_space_group = i_model.space_group_info()
i_model.show_summary()
else:
i_model = None
print("Target unit cell and space group:", file=out)
print(" ", self.params.target_unit_cell, file=out)
print(" ", self.params.target_space_group, file=out)
from xfel.command_line.cxi_merge import consistent_set_and_model
self.miller_set, self.i_model = consistent_set_and_model(
self.params, i_model)
self.frame_files = get_observations(self.params)
self.out = out
def start(self, interval=None, opts=None):
self.stop()
if self.opts.sdir!=opts.sdir or self.opts.ddir!=opts.ddir:
self._transferred = {}
self._bsslog_cache = {}
self.skipped = set()
self.opts = opts
self.log_out = multi_out()
self.log_out.register("stdout", sys.stdout)
self.log_out.register("file", open(os.path.join(self.opts.sdir, logfile_name), "a"))
self.log_out.write(time.strftime("# DATA SYNC GUI - Starting at %Y-%m-%d %H:%M:%S\n"))
self.opts.show(self.log_out)
self.log_out.write("\n")
self.log_out.flush()
self.keep_going = True
self.stop_now = False # for emergency stop
self.running = True
if interval is not None:
self.interval = interval
self.thread = threading.Thread(None, self.run)
self.thread.daemon = True
self.thread.start()
def get_input(args):
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = libtbx.phil.command_line.argument_interpreter(
master_phil=experimental_info.master_params,
home_scope="experiment")
print >> log, "#phil __OFF__"
print >> log, "=========================="
print >> log, " SDP "
print >> log, " SAXS Data Processing "
print >> log, "=========================="
print >> log
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = experimental_info.master_params.fetch(sources=phil_objects)
params = effective_params.extract()
print >> log, "#phil __ON__"
new_params = experimental_info.master_params.format(python_object=params)
new_params.show(out=log,expert_level=1)
print >> log, "#phil __END__"
run( params, log )
def run(self):
self.callback_start()
self._stdout = multi_out()
self._tmp_stdout = open(self.stdout_file, "w")
self._stdout.register("Communication log", self._tmp_stdout)
old_stdout = sys.stdout
sys.stdout = stdout_redirect(self)
import libtbx.callbacks
libtbx.call_back.register_handler(self.callback_wrapper)
try:
return_value = self.target()
except Abort: # FIXME why is this not working properly?
self.callback_abort()
except Exception as e:
print(type(e).__name__, file=sys.stderr)
if (type(e).__name__ == "Abort"):
self.callback_abort()
else:
if e.__class__.__module__ == "Boost.Python":
e = RuntimeError("Boost.Python.%s: %s" %
(e.__class__.__name__, str(e)))
elif hasattr(e, "reset_module"):
e.reset_module()
traceback_str = "\n".join(
traceback.format_tb(sys.exc_info()[2]))
self.callback_error(e, traceback_str)
else:
#time.sleep(1)
self.callback_final(return_value)
sys.stdout = old_stdout
def test_reference_model(mon_lib_srv, ener_lib, prefix="tst_reference_model"):
from mmtbx.geometry_restraints.torsion_restraints.tst_reference_model import \
model_raw_records, reference_raw_records
from mmtbx.geometry_restraints.torsion_restraints.reference_model import \
reference_model
import mmtbx.model
# mstream = StringIO()
from libtbx.utils import multi_out
mstream = multi_out()
mstream.register("stdout", sys.stdout)
mstream_file_name = "polder.log"
mstreamfile = open(mstream_file_name, "w")
mstream.register("logfile", mstreamfile)
work_params = mmtbx.model.manager.get_default_pdb_interpretation_params()
work_params.reference_model.enabled = True
work_params.reference_model.fix_outliers = False
pdb_inp = iotbx.pdb.input(lines=model_raw_records.split('\n'),
source_info=None)
model = mmtbx.model.manager(model_input=pdb_inp,
process_input=True,
pdb_interpretation_params=work_params)
reference_hierarchy_list = []
tmp_hierarchy = iotbx.pdb.input(
source_info=None,
lines=reference_raw_records.split('\n')).construct_hierarchy()
reference_hierarchy_list.append(tmp_hierarchy)
rm = reference_model(model=model,
reference_hierarchy_list=reference_hierarchy_list,
params=work_params.reference_model,
log=mstream)
assert rm.get_n_proxies() == 5, "Got %d, expected 5" % rm.get_n_proxies()
geometry, xrs = make_initial_grm(mon_lib_srv, ener_lib, model_raw_records)
geometry.adopt_reference_dihedral_manager(rm)
make_geo_pickle_unpickle(geometry, xrs, prefix)
def DataManager(datatypes=None, phil=None, logger=None):
'''
Function for dynamically creating a DataManager instance that supports a
specific set of datatypes.
All DataManager modules in iotbx/data_manager follow this pattern:
filename = <datatype>.py -> module name = iotbx.data_manager.<datatype>
DataManagerBase subclass name = <Datatype>DataManager
So for the models, the filename is iotbx/data_manager/model.py and in that
file, there should be a subclass of DataManagerBase named ModelDataManager
'''
if logger is None:
logger = multi_out()
# set default if necessary
if datatypes is None:
datatypes = default_datatypes
# get classes
modules = load_datatype_modules(datatypes)
manager_classes = []
for datatype in datatypes:
module_name = 'iotbx.data_manager.' + datatype
class_name = datatype.capitalize() + 'DataManager'
if '_' in datatype: # real_map becomes RealMapDataManager
class_name = ''.join(
tmp_str.capitalize()
for tmp_str in datatype.split('_')) + 'DataManager'
manager_classes.append(getattr(sys.modules[module_name], class_name))
# check inheritance and add datatypes if necessary
class_datatypes = set()
parent_classes = []
for manager_class in manager_classes:
if hasattr(manager_class, 'datatype'):
class_datatypes.add(manager_class.datatype)
# get full inheritance order and check
for parent_class in inspect.getmro(manager_class)[1:]:
if hasattr(parent_class, 'datatype'):
class_datatypes.add(parent_class.datatype)
parent_classes.append(parent_class)
try: # remove parent class and add back later
manager_classes.remove(parent_class)
except ValueError: # parent class already removed
pass
datatypes = list(class_datatypes)
# add mixin classes if necessary
mixin_classes = []
if 'real_map' in datatypes and 'map_coefficients' in datatypes:
importlib.import_module('.common', package='iotbx.data_manager')
mixin_classes.append(
getattr(sys.modules['iotbx.data_manager.common'], 'map_mixins'))
# construct new class and return instance
classes = tuple(manager_classes + parent_classes + mixin_classes)
data_manager_class = type('DataManager', classes, dict())
return data_manager_class(datatypes=datatypes, phil=phil, logger=logger)
def run (self) :
self.callback_start()
self._stdout = multi_out()
self._tmp_stdout = open(self.stdout_file, "w")
self._stdout.register("Communication log", self._tmp_stdout)
old_stdout = sys.stdout
sys.stdout = stdout_redirect(self)
import libtbx.callbacks
libtbx.call_back.register_handler(self.callback_wrapper)
try :
return_value = self.target()
except Abort : # FIXME why is this not working properly?
self.callback_abort()
except Exception, e :
print >> sys.stderr, type(e).__name__
if (type(e).__name__ == "Abort") :
self.callback_abort()
else :
if e.__class__.__module__ == "Boost.Python" :
e = RuntimeError("Boost.Python.%s: %s" % (e.__class__.__name__,
str(e)))
elif hasattr(e, "reset_module") :
e.reset_module()
traceback_str = "\n".join(traceback.format_tb(sys.exc_info()[2]))
self.callback_error(e, traceback_str)
def run_program(program_class=None,
custom_process_arguments=None,
args=None,
logger=None):
'''
Function for running programs using CCTBXParser and the program template
:param program_class: ProgramTemplate type (required)
:param custom_process_arguments:
Custom function to parse unknown arguments (optional)
:param args: list of command-line arguments (optional)
:param logger: logger (e.g. multi_out) for output (optional)
:rtype: whatever is returned from program_class.get_results()
'''
assert program_class is not None
if args is None:
args = sys.argv[1:]
# create logger
if logger is None:
logger = multi_out()
logger.register('stdout', sys.stdout)
logger.register('parser_log', StringIO())
# start timer
t = show_times(out=logger)
# create parser
parser = CCTBXParser(program_class=program_class,
custom_process_arguments=custom_process_arguments,
logger=logger)
namespace = parser.parse_args(args)
# start program
print('Starting job', file=logger)
print('=' * 79, file=logger)
task = program_class(parser.data_manager,
parser.working_phil.extract(),
master_phil=parser.master_phil,
logger=logger)
# validate inputs
task.validate()
# run program
task.run()
# clean up (optional)
task.clean_up()
# stop timer
print('', file=logger)
print('=' * 79, file=logger)
print('Job complete', file=logger)
t()
return task.get_results()
def initialize(self, args):
import iotbx.phil
from xfel.command_line.cxi_merge import master_phil
if ("--help" in args):
iotbx.phil.parse(master_phil).show(attributes_level=2)
return
phil = iotbx.phil.process_command_line(
args=args, master_string=master_phil).show()
work_params = phil.work.extract()
from xfel.merging.phil_validation import application
application(work_params)
if ((work_params.d_min is None) or (work_params.data is None)
or ((work_params.model is None)
and work_params.scaling.algorithm != "mark1")):
command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
raise Usage(command_name + " "
"d_min=4.0 "
"data=~/scratch/r0220/006/strong/ "
"model=3bz1_3bz2_core.pdb")
if ((work_params.rescale_with_average_cell)
and (not work_params.set_average_unit_cell)):
raise Usage(
"If rescale_with_average_cell=True, you must also specify " +
"set_average_unit_cell=True.")
if [
work_params.raw_data.sdfac_auto,
work_params.raw_data.sdfac_refine,
work_params.raw_data.errors_from_sample_residuals
].count(True) > 1:
raise Usage(
"Specify only one of sdfac_auto, sdfac_refine or errors_from_sample_residuals."
)
# Read Nat's reference model from an MTZ file. XXX The observation
# type is given as F, not I--should they be squared? Check with Nat!
log = open("%s.log" % work_params.output.prefix, "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
print >> out, "I model"
if work_params.model is not None:
from xfel.merging.general_fcalc import run as run_fmodel
i_model = run_fmodel(work_params)
work_params.target_unit_cell = i_model.unit_cell()
work_params.target_space_group = i_model.space_group_info()
i_model.show_summary()
else:
i_model = None
print >> out, "Target unit cell and space group:"
print >> out, " ", work_params.target_unit_cell
print >> out, " ", work_params.target_space_group
from xfel.command_line.cxi_merge import consistent_set_and_model
self.miller_set, self.i_model = consistent_set_and_model(
work_params, i_model)
self.work_params = work_params
self.frame_files = get_observations(work_params)
self.out = out
def set_log(prefix, i_current, i_total):
log = multi_out()
fo = open("%s.tls.%s_of_%s"%(prefix,str(i_current),i_total),"w")
log.register(
label = "log_buffer",
file_object = fo)
sys.stderr = log
return log
def run(args):
# create parser
logger = multi_out() #logging.getLogger('main')
logger.register('stdout', sys.stdout)
parser = CCTBXParser(program_class=atom_selection.Program,
custom_process_arguments=custom_args_proc,
logger=logger)
#
# Stick in additional keyword args
# They going to end up in namespace.cryst1_replacement_buffer_layer etc
# file_name is obsolet, parser takes care of it putting it in data manager
# inselections is going to be handled by custom_args_proc function
# because it is intended to be positional argument
#
# !!! This is done here for legacy support and illustrative purposes.
# Don't do it anywhere else, since we strive to have the same
# command-line flags across all programs, like --overwrite etc.
parser.add_argument(
"--cryst1-replacement-buffer-layer",
action="store",
type=float,
help="replace CRYST1 with pseudo unit cell covering the selected"
" atoms plus a surrounding buffer layer",
default=None)
parser.add_argument("--write-pdb-file",
"--write_pdb_file",
"--write_pdb-file",
"--write-pdb_file",
action="store",
help="write selected atoms to new PDB file",
default=None)
namespace = parser.parse_args(sys.argv[1:])
# start program
print('Starting job', file=logger)
print('=' * 79, file=logger)
task = atom_selection.Program(parser.data_manager,
parser.working_phil.extract(),
master_phil=parser.master_phil,
logger=logger)
# validate inputs
task.validate()
# run program
task.run()
# stop timer
print('', file=logger)
print('=' * 79, file=logger)
print('Job complete', file=logger)
show_total_time(out=logger)
def __call__ (self) :
if (self.log_file is not None) :
log = open(self.log_file, "w")
new_out = multi_out()
new_out.register("log", log)
new_out.register("stdout", sys.stdout)
sys.stdout = new_out
self._out = new_out
result = self.run()
easy_pickle.dump(self.file_name, result)
if (self._out is not None) and (not getattr(self._out, "closed", False)) :
self._out.flush()
# FIXME
#self._out.close()
return result
def run(args):
logger = multi_out() #logging.getLogger('main')
logger.register('stdout', sys.stdout)
# replace command-line parser
# ---------------------------------------------------------------------------
if (len(args) == 0):
show_usage(logger=logger)
sys.exit()
input_objects = iotbx.phil.process_command_line_with_files(
args=args,
master_phil=prepare_pdb_deposition.master_params,
pdb_file_def='input.model_file',
seq_file_def='input.sequence_file')
# get program settings
params = input_objects.work.extract()
# get files (will be handled by task.validate)
# already
# if (params.input.model_file is None):
# raise Sorry('One model file is required.')
# if (params.input.sequence_file is None):
# raise Sorry('One sequence file is required.')
pdb_input = iotbx.pdb.input(params.input.model_file)
model = mmtbx.model.manager(model_input=pdb_input, log=logger)
sequence = any_file(params.input.sequence_file)
sequence.check_file_type('seq')
sequence = sequence.file_object
# construct data manager
data_manager = DataManager()
data_manager.add_model(params.input.model_file, model)
data_manager.add_sequence(params.input.sequence_file, sequence)
# ---------------------------------------------------------------------------
# start program
task = prepare_pdb_deposition.Program(data_manager, params, logger=logger)
# validate inputs
task.validate()
# run program
task.run()
def __call__(self):
if (self.log_file is not None):
log = open(self.log_file, "w")
new_out = multi_out()
new_out.register("log", log)
new_out.register("stdout", sys.stdout)
sys.stdout = new_out
self._out = new_out
result = self.run()
easy_pickle.dump(self.file_name, result)
if (self._out
is not None) and (not getattr(self._out, "closed", False)):
self._out.flush()
# FIXME
#self._out.close()
return result
def __init__(self, data_manager, params, master_phil=None, logger=None):
'''
Common constructor for all programs
This is supposed to be lightweight. Custom initialization, if necessary,
should be handled by the custom_init function. Developers should not need to
override this function.
Parameters
----------
data_manager :
An instance of the DataManager (libtbx/data_manager.py) class containing
data structures from file input
params :
An instance of PHIL
logger :
Standard Python logger (from logging module), optional. A logger will be
created if it is not provided.
'''
self.data_manager = data_manager
self.master_phil = master_phil
self.params = params
self.logger = logger
if self.logger is None:
self.logger = multi_out()
# master_phil should be provided by CCTBXParser or GUI because of
# potential PHIL extensions
if self.master_phil is None:
self.master_phil = libtbx.phil.parse(self.master_phil_str,
process_includes=True)
# set DataManager defaults
if self.data_manager is not None:
self.data_manager.set_default_output_filename(
self.get_default_output_filename())
try:
self.data_manager.set_overwrite(self.params.output.overwrite)
except AttributeError:
pass
self.data_manager.set_program(self)
# optional initialization
self.custom_init()
def run (args, out=None) :
if (out is None) : out = sys.stdout
usage_string = """\
mmtbx.prune_model model.pdb data.mtz [options...]
Filters protein residues based on CC to 2mFo-DFc map and absolute
(sigma-scaled) values in 2mFo-DFc and mFo-DFc maps. For fast automatic
correction of MR solutions after initial refinement (ideally with rotamer
correction) to remove spurious loops and sidechains.
"""
from mmtbx.command_line import load_model_and_data
cmdline = load_model_and_data(
args=args,
master_phil=get_master_phil(),
out=out,
process_pdb_file=False,
create_fmodel=True)
params = cmdline.params
fmodel = cmdline.fmodel
if (params.output.file_name is None) :
base_name = os.path.basename(params.input.pdb.file_name[0])
params.output.file_name = os.path.splitext(base_name)[0] + "_pruned.pdb"
log_file = os.path.splitext(os.path.basename(params.output.file_name))[0] + \
".log"
log = open(log_file, "w")
out2 = multi_out()
out2.register("out", out)
out2.register("log", log)
map_helper = fmodel.electron_density_map()
f_map_coeffs = map_helper.map_coefficients(map_type="2mFo-DFc")
diff_map_coeffs = map_helper.map_coefficients(map_type="mFo-DFc")
model_map_coeffs = map_helper.map_coefficients(map_type="Fc")
result = prune_model(
f_map_coeffs=f_map_coeffs,
diff_map_coeffs=diff_map_coeffs,
model_map_coeffs=model_map_coeffs,
pdb_hierarchy=cmdline.pdb_hierarchy,
params=params.prune).process_residues(out=out2)
f = open(params.output.file_name, "w")
f.write("REMARK edited by mmtbx.prune_model\n")
f.write(cmdline.pdb_hierarchy.as_pdb_string(
crystal_symmetry=fmodel.xray_structure))
f.close()
log.close()
print >> out, "Wrote %s" % params.output.file_name
return params.output.file_name
def run(args, out=None):
if (out is None): out = sys.stdout
usage_string = """\
mmtbx.prune_model model.pdb data.mtz [options...]
Filters protein residues based on CC to 2mFo-DFc map and absolute
(sigma-scaled) values in 2mFo-DFc and mFo-DFc maps. For fast automatic
correction of MR solutions after initial refinement (ideally with rotamer
correction) to remove spurious loops and sidechains.
"""
from mmtbx.command_line import load_model_and_data
cmdline = load_model_and_data(args=args,
master_phil=get_master_phil(),
out=out,
process_pdb_file=False,
create_fmodel=True)
params = cmdline.params
fmodel = cmdline.fmodel
if (params.output.file_name is None):
base_name = os.path.basename(params.input.pdb.file_name[0])
params.output.file_name = os.path.splitext(
base_name)[0] + "_pruned.pdb"
log_file = os.path.splitext(os.path.basename(params.output.file_name))[0] + \
".log"
log = open(log_file, "w")
out2 = multi_out()
out2.register("out", out)
out2.register("log", log)
map_helper = fmodel.electron_density_map()
f_map_coeffs = map_helper.map_coefficients(map_type="2mFo-DFc")
diff_map_coeffs = map_helper.map_coefficients(map_type="mFo-DFc")
model_map_coeffs = map_helper.map_coefficients(map_type="Fc")
result = prune_model(f_map_coeffs=f_map_coeffs,
diff_map_coeffs=diff_map_coeffs,
model_map_coeffs=model_map_coeffs,
pdb_hierarchy=cmdline.pdb_hierarchy,
params=params.prune).process_residues(out=out2)
f = open(params.output.file_name, "w")
f.write("REMARK edited by mmtbx.prune_model\n")
f.write(
cmdline.pdb_hierarchy.as_pdb_string(
crystal_symmetry=fmodel.xray_structure))
f.close()
log.close()
print >> out, "Wrote %s" % params.output.file_name
return params.output.file_name
def DataManager(datatypes=None, phil=None, logger=None):
'''
Function for dynamically creating a DataManager instance that supports a
specific set of datatypes.
All DataManager modules in iotbx/data_manager follow this pattern:
filename = <datatype>.py -> module name = iotbx.data_manager.<datatype>
DataManagerBase subclass name = <Datatype>DataManager
So for the models, the filename is iotbx/data_manager/model.py and in that
file, there should be a subclass of DataManagerBase named ModelDataManager
'''
if (logger is None):
logger = multi_out()
# set default if necessary
if (datatypes is None):
datatypes = default_datatypes
# get classes
modules = load_datatype_modules(datatypes)
manager_classes = list()
for datatype in datatypes:
module_name = 'iotbx.data_manager.' + datatype
class_name = datatype.capitalize() + 'DataManager'
if ('_' in datatype): # real_map becomes RealMapDataManager
class_name = ''.join(
tmp_str.capitalize()
for tmp_str in datatype.split('_')) + 'DataManager'
manager_classes.append(getattr(sys.modules[module_name], class_name))
# check inheritance and add datatypes if necessary
class_datatypes = set()
for manager_class in manager_classes:
if (hasattr(manager_class, 'datatype')):
class_datatypes.add(manager_class.datatype)
# get full inheritance order and check
for parent_class in inspect.getmro(manager_class)[1:]:
if (hasattr(parent_class, 'datatype')):
class_datatypes.add(parent_class.datatype)
datatypes = list(class_datatypes)
# construct new class and return instance
data_manager_class = type('DataManager', tuple(manager_classes), dict())
return data_manager_class(datatypes=datatypes, phil=phil, logger=logger)
def run(params):
log_out = multi_out()
log_out.register("log", open(params.logfile, "w"), atexit_send_to=None)
log_out.register("stdout", sys.stdout)
libtbx.phil.parse(master_params_str).format(params).show(out=log_out, prefix=" ")
xac_files = read_path_list(params.lstin, only_exists=True, err_out=log_out)
if len(xac_files) == 0:
print >>log_out, "No (existing) files in the list: %s" % params.lstin
return
if params.method == "brehm_diederichs":
rb = BrehmDiederichs(xac_files, max_delta=params.max_delta,
d_min=params.d_min, min_ios=params.min_ios,
nproc=params.nproc, log_out=log_out)
elif params.method == "selective_breeding":
rb = KabschSelectiveBreeding(xac_files, max_delta=params.max_delta,
d_min=params.d_min, min_ios=params.min_ios,
nproc=params.nproc, log_out=log_out)
elif params.method == "reference":
rb = ReferenceBased(xac_files, params.reference_file, max_delta=params.max_delta,
d_min=params.d_min, min_ios=params.min_ios,
nproc=params.nproc, log_out=log_out)
else:
raise "Unknown method: %s" % params.method
rb.assign_operators()
new_files = rb.modify_xds_ascii_files()
lstout = os.path.splitext(os.path.basename(params.lstin))[0]+"_reindexed.lst"
ofs = open(lstout, "w")
ofs.write("\n".join(new_files)+"\n")
ofs.close()
print >>log_out, "Reindexing done. For merging, use %s instead!" % lstout
if params.method == "brehm_diederichs":
print >>log_out, """
CCTBX-implementation (by Richard Gildea) of the "algorithm 2" of the following paper was used.
For publication, please cite:
Brehm, W. and Diederichs, K. Breaking the indexing ambiguity in serial crystallography.
Acta Cryst. (2014). D70, 101-109
http://dx.doi.org/10.1107/S1399004713025431"""
elif params.method == "selective_breeding":
print >>log_out, """
def test_phil():
master_phil = libtbx.phil.parse(TestProgram.master_phil)
required_output_phil = libtbx.phil.parse(ProgramTemplate.output_phil_str)
master_phil.adopt_scope(required_output_phil)
params = master_phil.fetch(working_phil).extract()
logger = multi_out()
logger.register('stdout', sys.stdout)
test_program = TestProgram(None, params, master_phil, logger)
full_phil = libtbx.phil.parse(test_program.get_program_phil_str())
full = master_phil.fetch(full_phil).extract()
assert full.parameter_a == 'not None'
assert full.parameter_b == 5
assert full.parameter_c is None
assert 'parameter_c' in test_program.get_program_phil_str()
assert 'parameter_c' not in test_program.get_program_phil_str(True)
assert test_program.get_default_output_filename() == 'cctbx_program_000'
assert test_program.get_default_output_filename(prefix='abc') == 'abc_000'
assert test_program.get_default_output_filename(
suffix='abc') == 'cctbx_programabc_000'
assert test_program.get_default_output_filename(
serial=999) == 'cctbx_program_999'
assert test_program.get_default_output_filename(prefix='abc',
suffix='def',
serial=123) == 'abcdef_123'
test_program.params.output.prefix = 'prefix'
test_program.params.output.suffix = 'suffix'
test_program.params.output.serial = 7
assert test_program.get_default_output_filename() == 'prefixsuffix_007'
assert test_program.get_default_output_filename(
prefix='abc') == 'abcsuffix_007'
assert test_program.get_default_output_filename(
suffix='abc') == 'prefixabc_007'
assert test_program.get_default_output_filename(
serial=999) == 'prefixsuffix_999'
try:
test_program.get_default_output_filename(serial='abc')
except ValueError as e:
if str(e) != 'The serial argument should be an integer.':
raise
def run (args, out=None) :
if (out is None) : out = sys.stdout
from mmtbx.building.alternate_conformations import single_residue
import mmtbx.command_line
cmdline = mmtbx.command_line.load_model_and_data(
args=args,
master_phil=master_phil(),
process_pdb_file=True,
create_fmodel=True,
out=out,
usage_string="""\
mmtbx.build_alt_confs_simple model.pdb data.mtz [options]
Simple tool for building alternate conformations by real-space refinement into
difference density. Not intended for production use - use the program
mmtbx.build_alternate_conformations if you want refinement and post-processing.
""")
params = cmdline.params
validate_params(params)
log = multi_out()
log.register("stdout", out)
log_file_name = os.path.splitext(params.output.file_name)[0] + ".log"
logfile = open(log_file_name, "w")
log.register("logfile", logfile)
pdb_hierarchy, n_alternates = single_residue.build_cycle(
pdb_hierarchy = cmdline.pdb_hierarchy,
fmodel = cmdline.fmodel,
geometry_restraints_manager = cmdline.geometry,
params = params,
cif_objects=cmdline.cif_objects,
selection=params.selection,
nproc=params.nproc,
verbose=params.output.verbose,
debug=params.output.debug,
out=log)
# TODO real-space refinement of multi-conformer model
f = open(params.output.file_name, "w")
f.write(pdb_hierarchy.as_pdb_string(
crystal_symmetry=cmdline.fmodel.xray_structure))
f.close()
make_header("Building complete", out=out)
print >> log, ""
print >> log, "Wrote %s" % params.output.file_name
print >> log, "You MUST refine this model before using it!"
def run(args, out=None):
if (out is None): out = sys.stdout
from mmtbx.building.alternate_conformations import single_residue
import mmtbx.command_line
cmdline = mmtbx.command_line.load_model_and_data(args=args,
master_phil=master_phil(),
process_pdb_file=True,
create_fmodel=True,
out=out,
usage_string="""\
mmtbx.build_alt_confs_simple model.pdb data.mtz [options]
Simple tool for building alternate conformations by real-space refinement into
difference density. Not intended for production use - use the program
mmtbx.build_alternate_conformations if you want refinement and post-processing.
""")
params = cmdline.params
validate_params(params)
log = multi_out()
log.register("stdout", out)
log_file_name = os.path.splitext(params.output.file_name)[0] + ".log"
logfile = open(log_file_name, "w")
log.register("logfile", logfile)
pdb_hierarchy, n_alternates = single_residue.build_cycle(
pdb_hierarchy=cmdline.pdb_hierarchy,
fmodel=cmdline.fmodel,
geometry_restraints_manager=cmdline.geometry,
params=params,
cif_objects=cmdline.cif_objects,
selection=params.selection,
nproc=params.nproc,
verbose=params.output.verbose,
debug=params.output.debug,
out=log)
# TODO real-space refinement of multi-conformer model
f = open(params.output.file_name, "w")
f.write(
pdb_hierarchy.as_pdb_string(
crystal_symmetry=cmdline.fmodel.xray_structure))
f.close()
make_header("Building complete", out=out)
print >> log, ""
print >> log, "Wrote %s" % params.output.file_name
print >> log, "You MUST refine this model before using it!"
def run(args):
# processing command-line stuff, out of the object
log = multi_out()
log.register("stdout", sys.stdout)
if len(args) == 0:
format_usage_message(log)
return
inputs = mmtbx.utils.process_command_line_args(args=args,
master_params=master_params())
work_params = inputs.params.extract()
inputs.params.show(prefix=" ", out=log)
pdb_file_names = list(inputs.pdb_file_names)
if work_params.file_name is not None:
pdb_file_names += work_params.file_name
if len(pdb_file_names) == 0:
raise Sorry("No PDB file specified")
if work_params.output_prefix is None:
work_params.output_prefix = os.path.basename(pdb_file_names[0])
log_file_name = "%s.log" % work_params.output_prefix
logfile = open(log_file_name, "w")
log.register("logfile", logfile)
if work_params.loop_idealization.output_prefix is None:
work_params.loop_idealization.output_prefix = "%s_rama_fixed" % work_params.output_prefix
pdb_combined = iotbx.pdb.combine_unique_pdb_files(file_names=pdb_file_names)
pdb_input = iotbx.pdb.input(source_info=None,
lines=flex.std_string(pdb_combined.raw_records))
mi_object = model_idealization(
pdb_input=pdb_input,
cif_objects=inputs.cif_objects,
params=work_params,
log=log,
verbose=True)
mi_object.run()
mi_object.print_stat_comparison()
print >> log, "RMSD from starting model (backbone, all): %.4f, %.4f" % (
mi_object.get_rmsd_from_start(), mi_object.get_rmsd_from_start2())
mi_object.print_runtime()
# add hydrogens if needed ?
print >> log, "All done."
log.close()
def test_phil():
master_phil = libtbx.phil.parse(TestProgram.master_phil)
required_output_phil = libtbx.phil.parse(ProgramTemplate.output_phil_str)
master_phil.adopt_scope(required_output_phil)
params = master_phil.fetch(working_phil).extract()
logger = multi_out()
logger.register('stdout', sys.stdout)
test_program = TestProgram(None, params, master_phil, logger)
full_phil = libtbx.phil.parse(test_program.get_program_phil_str())
full = master_phil.fetch(full_phil).extract()
assert (full.parameter_a == 'not None')
assert (full.parameter_b == 5)
assert (full.parameter_c is None)
assert ('parameter_c' in test_program.get_program_phil_str())
assert ('parameter_c' not in test_program.get_program_phil_str(True))
assert (test_program.get_default_output_filename() == 'cctbx_program_000')
test_program.params.output.prefix = 'prefix'
test_program.params.output.suffix = 'suffix'
test_program.params.output.serial = 7
assert (test_program.get_default_output_filename() == 'prefixsuffix_007')
def run(args, out=sys.stdout):
log = multi_out()
log.register("stdout", out)
log_file_name = "hydrogen_parameterization.log"
logfile = open(log_file_name, "w")
log.register("logfile", logfile)
if (len(args) == 0):
print >>log, legend
return
print >> log, "phenix.hydrogen_parameterization is running..."
print >> log, "input parameters:\n", args
# parse through params --> switch off CDL or not
params_line = grand_master_phil_str
params = iotbx.phil.parse(
input_string=params_line, process_includes=True).extract()
params.pdb_interpretation.restraints_library.cdl=False
processed_args = mmtbx.utils.process_command_line_args(
args=args, log=null_out())
pdb_filename = processed_args.pdb_file_names[0]
mon_lib_srv = monomer_library.server.server()
ener_lib = monomer_library.server.ener_lib()
if pdb_filename is not None:
processed_pdb_file = monomer_library.pdb_interpretation.process(
params = params.pdb_interpretation,
mon_lib_srv = mon_lib_srv,
ener_lib = ener_lib,
file_name = pdb_filename,
force_symmetry = True)
pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy
xray_structure = processed_pdb_file.xray_structure()
if pdb_filename is not None:
pdb_str = pdb_hierarchy.as_pdb_string()
geometry_restraints = processed_pdb_file.geometry_restraints_manager(
show_energies = False)
restraints_manager = mmtbx.restraints.manager(
geometry = geometry_restraints,
normalization = False)
bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
sites_cart = xray_structure.sites_cart())
angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
hd_selection = xray_structure.hd_selection()
names = list(pdb_hierarchy.atoms().extract_name())
sites_cart = xray_structure.sites_cart()
#scatterers = xray_structure.scatterers()
atoms_list = list(pdb_hierarchy.atoms_with_labels())
print >>log, '\nNow determining connectivity table for H atoms...'
connectivity = hydrogen_connectivity.determine_H_neighbors(
geometry_restraints = geometry_restraints,
bond_proxies = bond_proxies_simple,
angle_proxies = angle_proxies,
hd_selection = hd_selection,
sites_cart = sites_cart)
print >>log, '\nNow determining the parameterization for H atoms...'
h_parameterization = get_h_parameterization(
connectivity = connectivity,
sites_cart = sites_cart,
idealize = False)
print >>log, '\nNow reconstructing H atoms...'
long_distance_list = []
unk_list = []
for ih in h_parameterization.keys():
residue = atoms_list[ih].resseq
hp = h_parameterization[ih]
h_obj = generate_H_positions(
sites_cart = sites_cart,
ih = ih,
para_info = hp)
if(h_obj.distance is not None):
print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
residue, 'distance:', h_obj.distance
if(h_obj.distance > 0.05):
long_distance_list.append(ih)
else:
print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', residue
unk_list.append(ih)
# some informative output for residues with unknown algorithm
print >>log, '*'*79
print >>log, 'Warning: The following atoms where not assigned an H type'
for ih in unk_list:
residue = atoms_list[ih].resseq
hp = h_parameterization[ih]
print >> log, 'atom:', names[ih], 'residue:', residue, \
'chain', atoms_list[ih].chain_id
print >>log, '*'*79
# some informative output for residues where position is NOT reproduced
# -> wronlgy assigned
print >>log, 'Warning: The position of the following H atoms was not reproduced'
for ih in long_distance_list:
residue = atoms_list[ih].resseq
hp = h_parameterization[ih]
h_obj = generate_H_positions(
sites_cart = sites_cart,
ih = ih,
para_info = hp)
if(h_obj.distance is not None and h_obj.distance > 0.05):
print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
residue, 'chain', atoms_list[ih].chain_id, 'distance:', h_obj.distance
print >>log, '*'*79
for ih in h_parameterization.keys():
hp = h_parameterization[ih]
print 'htype = ', hp.htype, 'a0 = ', hp.a0, 'a1 = ', hp.a1, 'a2 = ', hp.a2, \
'a = ', hp.a, 'b = ', hp.b, 'h = ', hp.h, 'chi = ', hp.chi, 'eps = ', hp.eps, \
'alpha = ', hp.alpha, 'dist_h =', hp.dist_h
def run(args):
if len(args)==0:
master_params.show(expert_level=100)
elif ( "--help" in args ):
print "no help available"
elif ( "--h" in args ):
print "no help available"
elif ( "--show_defaults" in args ):
master_params.show(expert_level=0)
elif ( "--show_defaults_all" in args ):
master_params.show(expert_level=10)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
log_plots = StringIO()
print >> log,"#phil __OFF__"
print >> log
print >> log, date_and_time()
print >> log
print >> log
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="scaling")
reflection_file = None
for arg in args:
command_line_params = None
arg_is_processed = False
if arg == '--quiet':
arg_is_processed = True
## The associated action with this keyword is implemented above
if (os.path.isfile(arg)): ## is this a file name?
## Check if this is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
except KeyboardInterrupt: raise
except Exception : pass
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
## Check if this file is a reflection file
if command_line_params is None:
reflection_file = reflection_file_reader.any_reflection_file(
file_name=arg, ensure_read_access=False)
if (reflection_file is not None):
reflection_file = arg
arg_is_processed = True
## If it is not a file, it must be a phil command
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown phil-file or phil-command:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file format or phil command: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
## Now please read in the reflections files
## get symmetry and cell data first please
## By default, the native cell and symmetry are used
## as reference
crystal_symmetry_nat = None
print params.scaling.input.xray_data.wavelength1.file_name
crystal_symmetry_nat = crystal_symmetry_from_any.extract_from(
file_name=params.scaling.input.xray_data.wavelength1.file_name)
if params.scaling.input.xray_data.space_group is None:
params.scaling.input.xray_data.space_group =\
crystal_symmetry_nat.space_group_info()
print >> log, "Using symmetry of native data"
if params.scaling.input.xray_data.unit_cell is None:
params.scaling.input.xray_data.unit_cell =\
crystal_symmetry_nat.unit_cell()
print >> log, "Using cell of native data"
## Check if a unit cell is defined
if params.scaling.input.xray_data.space_group is None:
raise Sorry("No space group defined")
if params.scaling.input.xray_data.unit_cell is None:
raise Sorry("No unit cell defined")
crystal_symmetry = crystal_symmetry = crystal.symmetry(
unit_cell = params.scaling.input.xray_data.unit_cell,
space_group_symbol = str(
params.scaling.input.xray_data.space_group) )
effective_params = master_params.fetch(sources=phil_objects)
new_params = master_params.format(python_object=params)
print >> log, "Effective parameters"
print >> log, "#phil __ON__"
new_params.show(out=log,expert_level=params.scaling.input.expert_level)
print >> log, "#phil __END__"
print >> log
## define a xray data server
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files=[])
## Read in native data and make appropriate selections
miller_array_w1 = None
miller_array_w1 = xray_data_server.get_xray_data(
file_name = params.scaling.input.xray_data.wavelength1.file_name,
labels = params.scaling.input.xray_data.wavelength1.labels,
ignore_all_zeros = True,
parameter_scope = 'scaling.input.SIR_scale.xray_data.native'
)
info_native = miller_array_w1.info()
miller_array_w1=miller_array_w1.map_to_asu().select(
miller_array_w1.indices()!=(0,0,0) )
miller_array_w1 = miller_array_w1.select(
miller_array_w1.data() > 0 )
## Convert to amplitudes
if (miller_array_w1.is_xray_intensity_array()):
miller_array_w1 = miller_array_w1.f_sq_as_f()
elif (miller_array_w1.is_complex_array()):
miller_array_w1 = abs(miller_array_w1)
if not miller_array_w1.is_real_array():
raise Sorry("miller_array_native is not a real array")
miller_array_w1.set_info(info = info_native)
## Read in derivative data and make appropriate selections
miller_array_w2 = None
miller_array_w2 = xray_data_server.get_xray_data(
file_name = params.scaling.input.xray_data.wavelength2.file_name,
labels = params.scaling.input.xray_data.wavelength2.labels,
ignore_all_zeros = True,
parameter_scope = 'scaling.input.SIR_scale.xray_data.derivative'
)
info_w2 = miller_array_w2.info()
miller_array_w2=miller_array_w2.map_to_asu().select(
miller_array_w2.indices()!=(0,0,0) )
miller_array_w2 = miller_array_w2.select(
miller_array_w2.data() > 0 )
## Convert to amplitudes
if (miller_array_w2.is_xray_intensity_array()):
miller_array_w2 = miller_array_w2.f_sq_as_f()
elif (miller_array_w2.is_complex_array()):
miller_array_w2 = abs(miller_array_w2)
if not miller_array_w2.is_real_array():
raise Sorry("miller_array_derivative is not a real array")
miller_array_w2.set_info(info = info_w2)
## Make sure we have anomalous diffs in both files
assert miller_array_w1.anomalous_flag()
assert miller_array_w2.anomalous_flag()
## Print info
print >> log
print >> log, "Wavelength 1"
print >> log, "============"
miller_array_w1.show_comprehensive_summary(f=log)
print >> log
w1_pre_scale = pre_scale.pre_scaler(
miller_array_w1,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_w1 = w1_pre_scale.x1.deep_copy()
del w1_pre_scale
print >> log
print >> log, "Wavelength 2"
print >> log, "============"
miller_array_w2.show_comprehensive_summary(f=log)
print >> log
w2_pre_scale = pre_scale.pre_scaler(
miller_array_w2,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_w2 = w2_pre_scale.x1.deep_copy()
del w2_pre_scale
print >> log
print >> log, "Checking for possible reindexing schemes"
print >> log, "----------------------------------------"
print >> log
print >> log, "Reindexing operator derived as described in:"
print >> log, "Grosse-Kunstleve, Afonine, Sauter & Adams. (2005)."
print >> log, " IUCr Computing Commission Newsletter 5."
print >> log
reindex_object = pair_analyses.reindexing(
set_a=miller_array_w1,
set_b=miller_array_w2,
out=log)
miller_array_w2 = reindex_object.select_and_transform()
miller_array_w2.map_to_asu()
print >> log
print >> log, "Relative scaling of 2-wavelength mad data"
print >> log, "-----------------------------------------"
print >> log
scaler = fa_estimation.combined_scaling(
miller_array_w1,
miller_array_w2,
params.scaling.input.scaling_strategy.iso_protocol)
miller_array_w1 = scaler.x1.deep_copy()
miller_array_w2 = scaler.x2.deep_copy()
del scaler
print >> log
print >> log, "Estimating f\" and f' ratios"
print >> log, "----------------------------"
print >> log
# now things are scaled see if we can guestimate the ratio
fdpratio = pair_analyses.f_double_prime_ratio(
miller_array_w1,
miller_array_w2)
fpfdpratio = pair_analyses.delta_f_prime_f_double_prime_ratio(
miller_array_w1,
miller_array_w2)
k1 = fdpratio.ratio
k2 = fpfdpratio.ratio
if k1 is not None:
print >> log
print >> log, " The estimate of f\"(w1)/f\"(w2) is %3.2f"\
%(fdpratio.ratio)
if k2 is not None:
print >> log, " The estimate of (f'(w1)-f'(w2))/f\"(w2) is %3.2f"\
%(fpfdpratio.ratio)
print >> log
print >> log, " The quality of these estimates depends to a large extend"
print >> log, " on the quality of the data. If user supplied values"
print >> log, " of f\" and f' are given, they will be used instead "
print >> log, " of the estimates."
print >> log
if params.scaling.input.xray_data.wavelength1.f_double_prime is not None:
if params.scaling.input.xray_data.wavelength2.f_double_prime is not None:
k1 = (params.scaling.input.xray_data.wavelength1.f_double_prime/
params.scaling.input.xray_data.wavelength2.f_double_prime)
print >> log, " Using user specified f\" values"
print >> log, " user specified f\"(w1)/f\"(w2) is %3.2f"\
%(k1)
print >> log
if params.scaling.input.xray_data.wavelength1.f_prime is not None:
if params.scaling.input.xray_data.wavelength2.f_prime is not None:
if params.scaling.input.xray_data.wavelength2.f_double_prime is not None:
k2 = (params.scaling.input.xray_data.wavelength1.f_prime-
params.scaling.input.xray_data.wavelength2.f_prime)\
/params.scaling.input.xray_data.wavelength2.f_double_prime
print >> log, " Using user specified f\" and f' values"
print >> log, " user specified f\"(w1)/f\"(w2) is %3.2f"\
%(k2)
print >> log
fa_gen = fa_estimation.twmad_fa_driver(miller_array_w1,
miller_array_w2,
k1,
k2,
params.scaling.input.fa_estimation)
print >> log
print >> log, "writing mtz file"
print >> log, "----------------"
print >> log
## Please write out the abs_delta_f array
fa = fa_gen.fa_values
mtz_dataset = fa.as_mtz_dataset(
column_root_label='F'+params.scaling.input.output.outlabel)
mtz_dataset.mtz_object().write(
file_name=params.scaling.input.output.hklout)
def run(args, command_name="phenix.twin_map_utils"):
log=sys.stdout
params=None
if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.twin_utils.input.model.file_name)
phil_xs=None
if ([params.twin_utils.input.unit_cell,
params.twin_utils.input.space_group]).count(None)<2:
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group )
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.twin_utils.input.unit_cell = combined_xs.unit_cell()
params.twin_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.twin_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.twin_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.twin_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.twin_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.twin_utils.input.unit_cell,
space_group_info=params.twin_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
free_flags = None
tmp_params = utils.data_and_flags_master_params().extract()
# insert proper values please
tmp_params.file_name = params.twin_utils.input.xray_data.file_name
tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
tmp_params.r_free_flags.file_name=params.twin_utils.input.xray_data.file_name
tmp_params.r_free_flags.label=params.twin_utils.input.xray_data.free_flag
tmp_object = utils.determine_data_and_flags( reflection_file_server = xray_data_server,
parameters = tmp_params, log=log )
miller_array = tmp_object.extract_data()
if miller_array.is_xray_intensity_array():
miller_array = miller_array.f_sq_as_f()
assert miller_array.is_xray_amplitude_array()
free_flags = tmp_object.extract_flags(data = miller_array)
print >> log
print >> log, "Attempting to extract Free R flags"
free_flags = free_flags.customized_copy( data = flex.bool( free_flags.data()==1 ) )
if free_flags is None:
free_flags = miller_array.generate_r_free_flags(use_lattice_symmetry=True)
assert miller_array.observation_type() is not None
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
print >> log
if miller_array.indices().size() == 0:
raise Sorry("No data available")
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.twin_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary(f=log)
print >> log
#----------------------------------------------------------------
# step 3: get the twin laws for this xs
twin_laws = twin_analyses.twin_laws(
miller_array,
lattice_symmetry_max_delta=\
params.twin_utils.parameters.twinning.max_delta,
out=log)
print >> log
print >> log, "Preliminary data analyses"
print >> log, "=========================="
twin_laws.show(out=log)
#---------
# step 3:
# make twin model managers for all twin laws
print >> log
print >> log, "Overall and bulk solvent scale paranmeters and twin fraction estimation"
print >> log, "======================================================================="
twin_models = []
operator_count = 0
if params.twin_utils.parameters.twinning.twin_law is not None:
tmp_law = sgtbx.rt_mx( params.twin_utils.parameters.twinning.twin_law )
tmp_law = twin_analyses.twin_law(tmp_law,None,None,None,None,None)
twin_laws.operators = [ tmp_law ]
for twin_law in twin_laws.operators:
operator_count += 1
operator_hkl = sgtbx.change_of_basis_op( twin_law.operator ).as_hkl()
twin_model = twin_f_model.twin_model_manager(
f_obs=miller_array,
r_free_flags = free_flags,
xray_structure=model,
twin_law = twin_law.operator,
detwin_mode = params.twin_utils.parameters.twinning.detwin_mode,
out=log)
print >> log, "--- bulk solvent scaling ---"
twin_model.update_solvent_and_scale(update_f_part1=False)
twin_model.r_values()
twin_model.target()
twin_model.show_k_sol_b_sol_b_cart_target()
twin_model.show_essential()
wfofc = twin_model.map_coefficients(map_type="mFo-DFc" )
wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc" )
grad = twin_model.map_coefficients(map_type="gradient" )
mtz_dataset = wtfofc.as_mtz_dataset(
column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = wfofc,
column_root_label = "DFWT"
)
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = grad,
column_root_label = "GRAD"
)
name = params.twin_utils.output.map_coeffs_root+"_"+str(operator_count)+".mtz"
print >> log
print >> log, "Writing %s for twin law %s"%(name,operator_hkl)
print >> log
mtz_dataset.mtz_object().write(
file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = twin_model.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(
file_name=name)
if len(twin_laws.operators)==0:
print >> log
print >> log, "No twin laws were found"
print >> log, "Performing maximum likelihood based bulk solvent scaling"
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
f_model_object.update_solvent_and_scale(out=log)
tfofc = f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
mtz_dataset = tfofc.as_mtz_dataset(
column_root_label="FWT")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = fofc,
column_root_label = "DELFWT"
)
name = params.twin_utils.output.map_coeffs_root+"_ML.mtz"
mtz_dataset.mtz_object().write(
file_name=name)
if params.twin_utils.output.obs_and_calc is not None:
# i want also a Fobs and Fmodel combined dataset please
mtz_dataset = miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = f_model_object.f_model(),
column_root_label="FMODEL")
name = params.twin_utils.output.obs_and_calc
mtz_dataset.mtz_object().write(
file_name=name)
print >> log
print >> log
print >> log, "All done \n"
logfile = open(params.twin_utils.output.logfile,'w')
print >> logfile, string_buffer.getvalue()
print >> log
def run(args):
phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
work_params = phil.work.extract()
from xfel.merging.phil_validation import application
application(work_params)
if ("--help" in args) :
libtbx.phil.parse(master_phil.show())
return
if ((work_params.d_min is None) or
(work_params.data is None) or
( (work_params.model is None) and work_params.scaling.algorithm != "mark1") ) :
raise Usage("cxi.merge "
"d_min=4.0 "
"data=~/scratch/r0220/006/strong/ "
"model=3bz1_3bz2_core.pdb")
if ((work_params.rescale_with_average_cell) and
(not work_params.set_average_unit_cell)) :
raise Usage("If rescale_with_average_cell=True, you must also specify "+
"set_average_unit_cell=True.")
if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
raise Usage("Cannot specify both sdfac_auto and sdfac_refine")
log = open("%s_%s.log" % (work_params.output.prefix,work_params.scaling.algorithm), "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
# Verify that the externally supplied isomorphous reference, if
# present, defines a suitable column of intensities, and exit with
# error if it does not. Then warn if it is necessary to generate
# Bijvoet mates. Failure to catch these issues here would lead to
# possibly obscure problems in cxi/cxi_cc.py later on.
try:
data_SR = mtz.object(work_params.scaling.mtz_file)
except RuntimeError:
pass
else:
array_SR = None
obs_labels = []
for array in data_SR.as_miller_arrays():
this_label = array.info().label_string().lower()
if array.observation_type() is not None:
obs_labels.append(this_label.split(',')[0])
if this_label.find('fobs')>=0:
array_SR = array.as_intensity_array()
break
if this_label.find('imean')>=0:
array_SR = array.as_intensity_array()
break
if this_label.find(work_params.scaling.mtz_column_F)==0:
array_SR = array.as_intensity_array()
break
if array_SR is None:
known_labels = ['fobs', 'imean', work_params.scaling.mtz_column_F]
raise Usage(work_params.scaling.mtz_file +
" does not contain any observations labelled [" +
", ".join(known_labels) +
"]. Please set scaling.mtz_column_F to one of [" +
",".join(obs_labels) + "].")
elif not work_params.merge_anomalous and not array_SR.anomalous_flag():
print >> out, "Warning: Preserving anomalous contributors, but %s " \
"has anomalous contributors merged. Generating identical Bijvoet " \
"mates." % work_params.scaling.mtz_file
# Read Nat's reference model from an MTZ file. XXX The observation
# type is given as F, not I--should they be squared? Check with Nat!
print >> out, "I model"
if work_params.model is not None:
from xfel.merging.general_fcalc import run
i_model = run(work_params)
work_params.target_unit_cell = i_model.unit_cell()
work_params.target_space_group = i_model.space_group_info()
i_model.show_summary()
else:
i_model = None
print >> out, "Target unit cell and space group:"
print >> out, " ", work_params.target_unit_cell
print >> out, " ", work_params.target_space_group
miller_set = symmetry(
unit_cell=work_params.target_unit_cell,
space_group_info=work_params.target_space_group
).build_miller_set(
anomalous_flag=not work_params.merge_anomalous,
d_max=work_params.d_max,
d_min=work_params.d_min / math.pow(
1 + work_params.unit_cell_length_tolerance, 1 / 3))
miller_set = miller_set.change_basis(
work_params.model_reindex_op).map_to_asu()
if i_model is not None:
matches = miller.match_indices(i_model.indices(), miller_set.indices())
assert not matches.have_singles()
miller_set = miller_set.select(matches.permutation())
# ---- Augment this code with any special procedures for x scaling
scaler = xscaling_manager(
miller_set=miller_set,
i_model=i_model,
params=work_params,
log=out)
scaler.scale_all()
if scaler.n_accepted == 0:
return None
# --- End of x scaling
scaler.uc_values = unit_cell_distribution()
for icell in xrange(len(scaler.frames["unit_cell"])):
if scaler.params.model is None:
scaler.uc_values.add_cell(
unit_cell=scaler.frames["unit_cell"][icell])
else:
scaler.uc_values.add_cell(
unit_cell=scaler.frames["unit_cell"][icell],
rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))
scaler.show_unit_cell_histograms()
if (work_params.rescale_with_average_cell) :
average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
average_cell = uctbx.unit_cell(list(average_cell_abc) +
list(work_params.target_unit_cell.parameters()[3:]))
work_params.target_unit_cell = average_cell
print >> out, ""
print >> out, "#" * 80
print >> out, "RESCALING WITH NEW TARGET CELL"
print >> out, " average cell: %g %g %g %g %g %g" % \
work_params.target_unit_cell.parameters()
print >> out, ""
scaler.reset()
scaler = xscaling_manager(
miller_set=miller_set,
i_model=i_model,
params=work_params,
log=out)
scaler.scale_all()
scaler.uc_values = unit_cell_distribution()
for icell in xrange(len(scaler.frames["unit_cell"])):
if scaler.params.model is None:
scaler.uc_values.add_cell(
unit_cell=scaler.frames["unit_cell"][icell])
else:
scaler.uc_values.add_cell(
unit_cell=scaler.frames["unit_cell"][icell],
rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))
scaler.show_unit_cell_histograms()
if False : #(work_params.output.show_plots) :
try :
plot_overall_completeness(completeness)
except Exception, e :
print "ERROR: can't show plots"
print " %s" % str(e)
def run (args) :
verbose = False
parser = OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="Turn on verbose output")
parser.add_option("--skip-tests", dest="skip_tests", action="store_true",
help="Don't import modules beginning with 'tst'")
options, args = parser.parse_args(args)
module_list = []
if (len(args) == 0) :
module_list.extend([ m.name for m in libtbx.env.module_list ])
else :
for arg in args :
assert (arg in libtbx.env.module_dict), arg
module_list.append(arg)
has_stdout = []
stdout_old = sys.stdout
for module_name in module_list :
if (module_name in ignore_modules) :
continue
try :
module = __import__(module_name)
except ImportError, e :
print >> sys.stderr, e
continue
assert len(module.__path__) == 1
mod_path = module.__path__[0]
path_fields = split_all(mod_path)
n_leading_dirs = len(path_fields) - 1
for dirname, dirnames, filenames in os.walk(mod_path) :
for file_name in filenames :
if file_name.endswith(".py") and (file_name != "libtbx_refresh.py") :
py_mod_name, ext = op.splitext(file_name)
if (ext != '.py') or ("." in py_mod_name) :
if (options.verbose) :
print >> sys.stderr, "skipping %s" % file_name
continue
py_path = split_all(dirname)[n_leading_dirs:]
import_name = ".".join(py_path)
if (not has_init_py(dirname)) or (import_name in ignore_modules) :
continue
top_level_module = py_path[0]
if (file_name != "__init__.py") :
import_name += "." + file_name[:-3]
if (import_name in ignore_modules) :
continue
elif ((file_name.startswith("tst_") or file_name.startswith("test_"))
and options.skip_tests) :
continue
if (options.verbose) :
print import_name
try :
sys.stdout = multi_out()
sys.stdout.register("stdout", stdout_old)
out = StringIO()
sys.stdout.register("stringio", out)
submodule = __import__(import_name)
if (out.getvalue() != '') :
has_stdout.append(import_name)
if (options.verbose) :
print >> sys.stderr, out.getvalue()
except ImportError, e :
print >> sys.stderr, e
finally :
sys.stdout = stdout_old
def run(params):
out = multi_out()
out.register("log", open(os.path.join(os.path.dirname(params.lstout), "multi_prep_merging.log"), "w"), atexit_send_to=None)
out.register("stdout", sys.stdout)
cell_params = libtbx.phil.parse(input_string=multi_check_cell_consistency.master_params_str).extract()
cell_params.topdir = params.topdir
cm = multi_check_cell_consistency.run(cell_params, out=out)
if len(cm.groups) == 0:
print "Oh, no. No data."
return
if params.space_group is not None and params.group_choice is None:
params.group_choice = 1 # maybe the largest group.
if params.group_choice < 1 or len(cm.groups) < params.group_choice:
print "Invalid group_choice=. Should be in 1..%d" % len(cm.groups)
return
possible_sgs = set(map(lambda x: cm.symms[x].space_group(), cm.groups[params.group_choice-1]))
if params.space_group is None:
print "Please specify space_group=. The possible choice is:"
print "\n".join(map(lambda x: x.info().symbol_and_number(), possible_sgs))
return
try:
if sgtbx.space_group_info(params.space_group).group() not in possible_sgs:
print "Invalid space group choice. The possible choice is:"
print "\n".join(map(lambda x: x.info().symbol_and_number(), possible_sgs))
return
except RuntimeError:
print "Invalid space group name or number (%s)" % params.space_group
return
symms = map(lambda i: cm.symms[i], cm.groups[params.group_choice-1])
dirs = map(lambda i: cm.dirs[i], cm.groups[params.group_choice-1])
sgnum = sgtbx.space_group_info(params.space_group).group().type().number()
# 1. Scale with specified symmetry
rescale_with_specified_symm(params.topdir, dirs, symms, out, sgnum=sgnum)
# 2. Resolve reindexing problem
# TODO: reconstruct unit cell by averaging
reference_symm = filter(lambda x:x.reflection_intensity_symmetry(False).space_group_info().type().number()==sgnum_laue, symms)[0]
cosets = reindex.reindexing_operators(reference_symm, reference_symm)
reidx_ops = cosets.combined_cb_ops()
print " Reference symmetry:", reference_symm.unit_cell(), reference_symm.space_group_info().symbol_and_number(),
print " Possible reindex operators:", map(lambda x: str(x.as_hkl()), reidx_ops)
if len(reidx_ops) == 1:
print " No reindexing problem exists."
elif params.reference_for_reindex is not None:
resolve_indexing_ambiguity_using_reference(dirs, reidx_ops, params.reference_for_reindex)
else:
resolve_indexing_ambiguity(dirs, reidx_ops)
ofs = open(params.lstout, "w")
for wd in dirs:
xas_full = os.path.join(wd, "XDS_ASCII_fullres.HKL")
if os.path.isfile(xas_full): # if _fullres.HKL exists, use it.
ofs.write("%s\n" % xas_full)
else:
ofs.write("%s\n" % os.path.join(wd, "XDS_ASCII.HKL"))
def run(args):
phil = iotbx.phil.process_command_line(args=args,
master_string=master_phil).show()
work_params = phil.work.extract()
from xfel.merging.phil_validation import application, samosa
application(work_params)
samosa(work_params)
if ("--help" in args):
libtbx.phil.parse(master_phil.show())
return
if ((work_params.d_min is None) or (work_params.data is None)):
command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
raise Usage(command_name + " "
"d_min=4.0 "
"data=~/scratch/r0220/006/strong/ "
"model=3bz1_3bz2_core.pdb")
if ((work_params.rescale_with_average_cell)
and (not work_params.set_average_unit_cell)):
raise Usage(
"If rescale_with_average_cell=True, you must also specify " +
"set_average_unit_cell=True.")
if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
raise Usage("Cannot specify both sdfac_auto and sdfac_refine")
# Read Nat's reference model from an MTZ file. XXX The observation
# type is given as F, not I--should they be squared? Check with Nat!
log = open("%s.log" % work_params.output.prefix, "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
print >> out, "I model"
if work_params.model is not None:
from xfel.merging.general_fcalc import run
i_model = run(work_params)
work_params.target_unit_cell = i_model.unit_cell()
work_params.target_space_group = i_model.space_group_info()
i_model.show_summary()
else:
i_model = None
print >> out, "Target unit cell and space group:"
print >> out, " ", work_params.target_unit_cell
print >> out, " ", work_params.target_space_group
miller_set, i_model = consistent_set_and_model(work_params, i_model)
frame_files = get_observations(work_params)
scaler = scaling_manager(miller_set=miller_set,
i_model=i_model,
params=work_params,
log=out)
scaler.scale_all(frame_files)
if scaler.n_accepted == 0:
return None
scaler.show_unit_cell_histograms()
if (work_params.rescale_with_average_cell):
average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
average_cell = uctbx.unit_cell(
list(average_cell_abc) +
list(work_params.target_unit_cell.parameters()[3:]))
work_params.target_unit_cell = average_cell
print >> out, ""
print >> out, "#" * 80
print >> out, "RESCALING WITH NEW TARGET CELL"
print >> out, " average cell: %g %g %g %g %g %g" % \
work_params.target_unit_cell.parameters()
print >> out, ""
scaler.reset()
scaler.scale_all(frame_files)
scaler.show_unit_cell_histograms()
if False: #(work_params.output.show_plots) :
try:
plot_overall_completeness(completeness)
except Exception as e:
print "ERROR: can't show plots"
print " %s" % str(e)
print >> out, "\n"
# Sum the observations of I and I/sig(I) for each reflection.
sum_I = flex.double(miller_set.size(), 0.)
sum_I_SIGI = flex.double(miller_set.size(), 0.)
for i in xrange(miller_set.size()):
index = miller_set.indices()[i]
if index in scaler.ISIGI:
for t in scaler.ISIGI[index]:
sum_I[i] += t[0]
sum_I_SIGI[i] += t[1]
miller_set_avg = miller_set.customized_copy(
unit_cell=work_params.target_unit_cell)
table1 = show_overall_observations(obs=miller_set_avg,
redundancy=scaler.completeness,
summed_wt_I=scaler.summed_wt_I,
summed_weight=scaler.summed_weight,
ISIGI=scaler.ISIGI,
n_bins=work_params.output.n_bins,
title="Statistics for all reflections",
out=out,
work_params=work_params)
print >> out, ""
n_refl, corr = ((scaler.completeness > 0).count(True), 0)
print >> out, "\n"
table2 = show_overall_observations(
obs=miller_set_avg,
redundancy=scaler.summed_N,
summed_wt_I=scaler.summed_wt_I,
summed_weight=scaler.summed_weight,
ISIGI=scaler.ISIGI,
n_bins=work_params.output.n_bins,
title="Statistics for reflections where I > 0",
out=out,
work_params=work_params)
#from libtbx import easy_pickle
#easy_pickle.dump(file_name="stats.pickle", obj=stats)
#stats.report(plot=work_params.plot)
#miller_counts = miller_set_p1.array(data=stats.counts.as_double()).select(
# stats.counts != 0)
#miller_counts.as_mtz_dataset(column_root_label="NOBS").mtz_object().write(
# file_name="nobs.mtz")
if work_params.data_subsubsets.subsubset is not None and work_params.data_subsubsets.subsubset_total is not None:
easy_pickle.dump(
"scaler_%d.pickle" % work_params.data_subsubsets.subsubset, scaler)
print >> out, ""
mtz_file, miller_array = scaler.finalize_and_save_data()
#table_pickle_file = "%s_graphs.pkl" % work_params.output.prefix
#easy_pickle.dump(table_pickle_file, [table1, table2])
loggraph_file = os.path.abspath("%s_graphs.log" %
work_params.output.prefix)
f = open(loggraph_file, "w")
f.write(table1.format_loggraph())
f.write("\n")
f.write(table2.format_loggraph())
f.close()
result = scaling_result(miller_array=miller_array,
plots=scaler.get_plot_statistics(),
mtz_file=mtz_file,
loggraph_file=loggraph_file,
obs_table=table1,
all_obs_table=table2,
n_reflections=n_refl,
overall_correlation=corr)
easy_pickle.dump("%s.pkl" % work_params.output.prefix, result)
return result
def simul_utils(args):
if len(args) == 0:
print_help()
elif ("--help" in args):
print_help()
elif ("--h" in args):
print_help()
elif ("-h" in args):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
print("#phil __OFF__", file=log)
print("======================", file=log)
print(" SIMUL ", file=log)
print("A data simulation tool", file=log)
print("======================", file=log)
print(file=log)
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print("##----------------------------------------------##",
file=log)
print("## Unknown file or keyword:", arg, file=log)
print("##----------------------------------------------##",
file=log)
print(file=log)
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group)
combined_xs = select_crystal_symmetry(None, phil_xs, [pdb_xs],
[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.simul_utils.input.unit_cell = combined_xs.unit_cell()
params.simul_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print("#phil __ON__", file=log)
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print("#phil __END__", file=log)
if params.simul_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.simul_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.simul_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.simul_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group)
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name=params.simul_utils.input.xray_data.file_name,
labels=params.simul_utils.input.xray_data.labels,
ignore_all_zeros=True,
parameter_scope='simul_utils.input.xray_data',
parameter_name='labels')
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(miller_array.indices() != (0, 0, 0))
miller_array = miller_array.select(miller_array.data() > 0)
if miller_array.sigmas() is not None:
miller_array = miller_array.select(miller_array.sigmas() > 0)
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
print(file=log)
print("Summary info of observed data", file=log)
print("=============================", file=log)
miller_array.show_summary(f=log)
print(file=log)
free_flags = miller_array.generate_r_free_flags()
#--------------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.simul_utils.input.model.file_name
).xray_structure_simple(crystal_symmetry=phil_xs, )
print("Atomic model summary", file=log)
print("====================", file=log)
model.show_summary()
print(file=log)
#-------------------------------------------------------------------
# Step 3: make an F_model object to get model phases and amplitudes
#
print("Performing bulk solvent scaling", file=log)
print("===============================", file=log)
print(file=log)
print(file=log)
f_model_object = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=model)
f_model_object.update_all_scales(log=log)
fmodel = abs(
f_model_object.f_model()).set_observation_type(miller_array)
mockfmodel = None
if params.simul_utils.input.mock_model.file_name is not None:
print("Reading in mock model", file=log)
print("=====================", file=log)
print(file=log)
print(file=log)
mock_model = pdb.input(file_name=params.simul_utils.input.
mock_model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
mock_f_model = f_model.manager(f_obs=miller_array,
r_free_flags=free_flags,
xray_structure=mock_model)
mock_f_model.update(k_sol=f_model_object.k_sol(),
b_sol=f_model_object.b_sol(),
b_cart=f_model_object.b_cart())
mockfmodel = abs(
mock_f_model.f_model()).set_observation_type(miller_array)
else:
mockfmodel = fmodel.deep_copy()
print("Making new data", file=log)
print("===============", file=log)
print(file=log)
print(file=log)
new_data_builder = error_swap(miller_array, fmodel, mockfmodel)
new_data = new_data_builder.new_obs
# we now have to write the data actually
print("Writing new data set", file=log)
print("====================", file=log)
mtz_dataset = new_data.as_mtz_dataset(column_root_label="FOBS")
mtz_dataset.mtz_object().write(
file_name=params.simul_utils.output.hklout)
def run(self, args, command_name, out=sys.stdout):
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description='Example: %s data.mtz data.mtz ref_model.pdb' %
command_name).option(
None,
"--show_defaults",
action="store_true",
help="Show list of parameters.")).process(args=args)
cif_file = None
processed_args = utils.process_command_line_args(
args=args, log=sys.stdout, master_params=master_phil)
params = processed_args.params
if (params is None): params = master_phil
self.params = params.extract().ensemble_probability
pdb_file_names = processed_args.pdb_file_names
if len(pdb_file_names) != 1:
raise Sorry("Only one PDB structure may be used")
pdb_file = file_reader.any_file(pdb_file_names[0])
self.log = multi_out()
self.log.register(label="stdout", file_object=sys.stdout)
self.log.register(label="log_buffer",
file_object=StringIO(),
atexit_send_to=None)
sys.stderr = self.log
log_file = open(
pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.log', "w")
self.log.replace_stringio(old_label="log_buffer",
new_label="log",
new_file_object=log_file)
utils.print_header(command_name, out=self.log)
params.show(out=self.log)
#
f_obs = None
r_free_flags = None
reflection_files = processed_args.reflection_files
if self.params.fobs_vs_fcalc_post_nll:
if len(reflection_files) == 0:
raise Sorry(
"Fobs from input MTZ required for fobs_vs_fcalc_post_nll")
if len(reflection_files) > 0:
crystal_symmetry = processed_args.crystal_symmetry
print('Reflection file : ',
processed_args.reflection_file_names[0],
file=self.log)
utils.print_header("Model and data statistics", out=self.log)
rfs = reflection_file_server(
crystal_symmetry=crystal_symmetry,
reflection_files=processed_args.reflection_files,
log=self.log)
parameters = extract_xtal_data.data_and_flags_master_params(
).extract()
determine_data_and_flags_result = extract_xtal_data.run(
reflection_file_server=rfs,
parameters=parameters,
data_parameter_scope="refinement.input.xray_data",
flags_parameter_scope="refinement.input.xray_data.r_free_flags",
data_description="X-ray data",
keep_going=True,
log=self.log)
f_obs = determine_data_and_flags_result.f_obs
number_of_reflections = f_obs.indices().size()
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(
data=flex.bool(f_obs.data().size(), False))
# process PDB
pdb_file.assert_file_type("pdb")
#
pdb_in = hierarchy.input(file_name=pdb_file.file_name)
ens_pdb_hierarchy = pdb_in.construct_hierarchy()
ens_pdb_hierarchy.atoms().reset_i_seq()
ens_pdb_xrs_s = pdb_in.input.xray_structures_simple()
number_structures = len(ens_pdb_xrs_s)
print('Number of structure in ensemble : ',
number_structures,
file=self.log)
# Calculate sigmas from input map only
if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None:
# process MTZ
input_file = file_reader.any_file(
self.params.ensemble_sigma_map_input)
if input_file.file_type == "hkl":
if input_file.file_object.file_type() != "ccp4_mtz":
raise Sorry("Only MTZ format accepted for map input")
else:
mtz_file = input_file
else:
raise Sorry("Only MTZ format accepted for map input")
miller_arrays = mtz_file.file_server.miller_arrays
map_coeffs_1 = miller_arrays[0]
#
xrs_list = []
for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
# get sigma levels from ensemble fc for each structure
xrs = get_map_sigma(ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
map_coeffs_1=map_coeffs_1,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
log=self.log)
xrs_list.append(xrs)
# write ensemble pdb file, occupancies as sigma level
filename = pdb_file_names[0].split('/')[-1].replace(
'.pdb',
'') + '_vs_' + self.params.ensemble_sigma_map_input.replace(
'.mtz', '') + '_pensemble.pdb'
write_ensemble_pdb(filename=filename,
xrs_list=xrs_list,
ens_pdb_hierarchy=ens_pdb_hierarchy)
# Do full analysis vs Fobs
else:
model_map_coeffs = []
fmodel = None
# Get <fcalc>
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
ens_pdb_xrs.set_occupancies(1.0)
if model == 0:
# If mtz not supplied get fobs from xray structure...
# Use input Fobs for scoring against nll
if self.params.fobs_vs_fcalc_post_nll:
dummy_fobs = f_obs
else:
if f_obs == None:
if self.params.fcalc_high_resolution == None:
raise Sorry(
"Please supply high resolution limit or input mtz file."
)
dummy_dmin = self.params.fcalc_high_resolution
dummy_dmax = self.params.fcalc_low_resolution
else:
print(
'Supplied mtz used to determine high and low resolution cuttoffs',
file=self.log)
dummy_dmax, dummy_dmin = f_obs.d_max_min()
#
dummy_fobs = abs(
ens_pdb_xrs.structure_factors(
d_min=dummy_dmin).f_calc())
dummy_fobs.set_observation_type_xray_amplitude()
# If mtz supplied, free flags are over written to prevent array size error
r_free_flags = dummy_fobs.array(
data=flex.bool(dummy_fobs.data().size(), False))
#
fmodel = utils.fmodel_simple(
scattering_table="wk1995",
xray_structures=[ens_pdb_xrs],
f_obs=dummy_fobs,
target_name='ls',
bulk_solvent_and_scaling=False,
r_free_flags=r_free_flags)
f_calc_ave = fmodel.f_calc().array(
data=fmodel.f_calc().data() * 0).deep_copy()
# XXX Important to ensure scale is identical for each model and <model>
fmodel.set_scale_switch = 1.0
f_calc_ave_total = fmodel.f_calc().data().deep_copy()
else:
fmodel.update_xray_structure(xray_structure=ens_pdb_xrs,
update_f_calc=True,
update_f_mask=False)
f_calc_ave_total += fmodel.f_calc().data().deep_copy()
print('Model :', model + 1, file=self.log)
print("\nStructure vs real Fobs (no bulk solvent or scaling)",
file=self.log)
print('Rwork : %5.4f ' % fmodel.r_work(),
file=self.log)
print('Rfree : %5.4f ' % fmodel.r_free(),
file=self.log)
print('K1 : %5.4f ' % fmodel.scale_k1(),
file=self.log)
fcalc_edm = fmodel.electron_density_map()
fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type='Fc')
fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset(
column_root_label='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_mtz_dataset.mtz_object().write(
file_name=str(model + 1) + "_Fc.mtz")
model_map_coeffs.append(fcalc_map_coeffs.deep_copy())
fmodel.update(f_calc=f_calc_ave.array(f_calc_ave_total /
number_structures))
print("\nEnsemble vs real Fobs (no bulk solvent or scaling)",
file=self.log)
print('Rwork : %5.4f ' % fmodel.r_work(), file=self.log)
print('Rfree : %5.4f ' % fmodel.r_free(), file=self.log)
print('K1 : %5.4f ' % fmodel.scale_k1(), file=self.log)
# Get <Fcalc> map
fcalc_ave_edm = fmodel.electron_density_map()
fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients(
map_type='Fc').deep_copy()
fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset(
column_root_label='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_ave_mtz_dataset.mtz_object().write(file_name="aveFc.mtz")
fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map()
fcalc_ave_map_coeffs.apply_volume_scaling()
fcalc_ave_map_data = fcalc_ave_map_coeffs.real_map_unpadded()
fcalc_ave_map_stats = maptbx.statistics(fcalc_ave_map_data)
print("<Fcalc> Map Stats :", file=self.log)
fcalc_ave_map_stats.show_summary(f=self.log)
offset = fcalc_ave_map_stats.min()
model_neg_ll = []
number_previous_scatters = 0
# Run through structure list again and get probability
xrs_list = []
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
if self.params.verbose:
print('\n\nModel : ',
model + 1,
file=self.log)
# Get model atom sigmas vs Fcalc
fcalc_map = model_map_coeffs[model].fft_map()
fcalc_map.apply_volume_scaling()
fcalc_map_data = fcalc_map.real_map_unpadded()
fcalc_map_stats = maptbx.statistics(fcalc_map_data)
if self.params.verbose:
print("Fcalc map stats :", file=self.log)
fcalc_map_stats.show_summary(f=self.log)
xrs = get_map_sigma(
ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
fft_map_1=fcalc_map,
model_i=model,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
number_previous_scatters=number_previous_scatters,
log=self.log)
fcalc_sigmas = xrs.scatterers().extract_occupancies()
del fcalc_map
# Get model atom sigmas vs <Fcalc>
xrs = get_map_sigma(
ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
fft_map_1=fcalc_ave_map_coeffs,
model_i=model,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
number_previous_scatters=number_previous_scatters,
log=self.log)
### For testing other residue averaging options
#print xrs.residue_selections
fcalc_ave_sigmas = xrs.scatterers().extract_occupancies()
# Probability of model given <model>
prob = fcalc_ave_sigmas / fcalc_sigmas
# XXX debug option
if False:
for n, p in enumerate(prob):
print(' {0:5d} {1:5.3f}'.format(n, p), file=self.log)
# Set probabilty between 0 and 1
# XXX Make Histogram / more stats
prob_lss_zero = flex.bool(prob <= 0)
prob_grt_one = flex.bool(prob > 1)
prob.set_selected(prob_lss_zero, 0.001)
prob.set_selected(prob_grt_one, 1.0)
xrs.set_occupancies(prob)
xrs_list.append(xrs)
sum_neg_ll = sum(-flex.log(prob))
model_neg_ll.append((sum_neg_ll, model))
if self.params.verbose:
print('Model probability stats :', file=self.log)
print(prob.min_max_mean().show(), file=self.log)
print(' Count < 0.0 : ',
prob_lss_zero.count(True),
file=self.log)
print(' Count > 1.0 : ',
prob_grt_one.count(True),
file=self.log)
# For averaging by residue
number_previous_scatters += ens_pdb_xrs.sites_cart().size()
# write ensemble pdb file, occupancies as sigma level
write_ensemble_pdb(
filename=pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.pdb',
xrs_list=xrs_list,
ens_pdb_hierarchy=ens_pdb_hierarchy)
# XXX Test ordering models by nll
# XXX Test removing nth percentile atoms
if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll:
for percentile in [1.0, 0.975, 0.95, 0.9, 0.8, 0.6, 0.2]:
model_neg_ll = sorted(model_neg_ll)
f_calc_ave_total_reordered = None
print_list = []
for i_neg_ll in model_neg_ll:
xrs = xrs_list[i_neg_ll[1]]
nll_occ = xrs.scatterers().extract_occupancies()
# Set q=0 nth percentile atoms
sorted_nll_occ = sorted(nll_occ, reverse=True)
number_atoms = len(sorted_nll_occ)
percentile_prob_cutoff = sorted_nll_occ[
int(number_atoms * percentile) - 1]
cutoff_selections = flex.bool(
nll_occ < percentile_prob_cutoff)
cutoff_nll_occ = flex.double(nll_occ.size(),
1.0).set_selected(
cutoff_selections,
0.0)
#XXX Debug
if False:
print('\nDebug')
for x in range(len(cutoff_selections)):
print(cutoff_selections[x], nll_occ[x],
cutoff_nll_occ[x])
print(percentile)
print(percentile_prob_cutoff)
print(cutoff_selections.count(True))
print(cutoff_selections.size())
print(cutoff_nll_occ.count(0.0))
print('Count q = 1 : ',
cutoff_nll_occ.count(1.0))
print('Count scatterers size : ',
cutoff_nll_occ.size())
xrs.set_occupancies(cutoff_nll_occ)
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True,
update_f_mask=True)
if f_calc_ave_total_reordered == None:
f_calc_ave_total_reordered = fmodel.f_calc().data(
).deep_copy()
f_mask_ave_total_reordered = fmodel.f_masks(
)[0].data().deep_copy()
cntr = 1
else:
f_calc_ave_total_reordered += fmodel.f_calc().data(
).deep_copy()
f_mask_ave_total_reordered += fmodel.f_masks(
)[0].data().deep_copy()
cntr += 1
fmodel.update(
f_calc=f_calc_ave.array(
f_calc_ave_total_reordered / cntr).deep_copy(),
f_mask=f_calc_ave.array(
f_mask_ave_total_reordered / cntr).deep_copy())
# Update solvent and scale
# XXX Will need to apply_back_trace on latest version
fmodel.set_scale_switch = 0
fmodel.update_all_scales()
# Reset occ for outout
xrs.set_occupancies(nll_occ)
# k1 updated vs Fobs
if self.params.fobs_vs_fcalc_post_nll:
print_list.append([
cntr, i_neg_ll[0], i_neg_ll[1],
fmodel.r_work(),
fmodel.r_free()
])
# Order models by nll and print summary
print(
'\nModels ranked by nll <Fcalc> R-factors recalculated',
file=self.log)
print('Percentile cutoff : {0:5.3f}'.format(percentile),
file=self.log)
xrs_list_sorted_nll = []
print(' | NLL <Rw> <Rf> Ens Model',
file=self.log)
for info in print_list:
print(' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'.
format(
info[0],
info[1],
info[3],
info[4],
info[2] + 1,
),
file=self.log)
xrs_list_sorted_nll.append(xrs_list[info[2]])
# Output nll ordered ensemble
write_ensemble_pdb(
filename='nll_ordered_' +
pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.pdb',
xrs_list=xrs_list_sorted_nll,
ens_pdb_hierarchy=ens_pdb_hierarchy)
def run (args,
out=sys.stdout,
program_name="phenix.molprobity",
ignore_missing_modules=False,
return_input_objects=False) : # for testing
rotarama_dir = libtbx.env.find_in_repositories(
relative_path="chem_data/rotarama_data",
test=os.path.isdir)
if (rotarama_dir is None) :
raise ImportError("Rotamer and Ramachandran distributions not available; "+
"you will need these to run MolProbity.")
elif (((not libtbx.env.has_module("reduce")) or
(not libtbx.env.has_module("probe"))) and
(not ignore_missing_modules)) :
raise ImportError("Reduce and/or Probe not configured.")
import mmtbx.validation.molprobity
import mmtbx.command_line
cmdline = mmtbx.command_line.load_model_and_data(
args=args,
master_phil=get_master_phil(),
require_data=False,
create_fmodel=True,
process_pdb_file=True,
usage_string=usage_string,
prefer_anomalous=True,
out=out)
params = cmdline.params
fmodel = cmdline.fmodel
if (params.output.maps is Auto) and (fmodel is not None) :
params.output.maps = True
elif (params.output.maps == True) and (fmodel is None) :
raise Sorry("Map output requires experimental data.")
if (params.molprobity.keep_hydrogens is Auto) :
params.molprobity.keep_hydrogens = \
(params.input.scattering_table == "neutron")
header_info = mmtbx.validation.molprobity.pdb_header_info(
pdb_file=params.input.pdb.file_name[0],
pdb_hierarchy=cmdline.pdb_hierarchy)
pdb_prefix = os.path.splitext(os.path.basename(
params.input.pdb.file_name[0]))[0]
if (params.output.prefix is None) :
params.output.prefix = "molprobity"
probe_file = None
if (params.output.probe_dots) or (params.output.kinemage) :
probe_file = params.output.prefix + "_probe.txt"
raw_data = cmdline.raw_data
# check map parameters
from mmtbx.real_space_correlation import check_map_file
check_map_file(None, params.input.maps)
validation = mmtbx.validation.molprobity.molprobity(
pdb_hierarchy=cmdline.pdb_hierarchy,
xray_structure=cmdline.xray_structure,
fmodel=fmodel,
flags=params.molprobity.flags,
crystal_symmetry=cmdline.crystal_symmetry,
sequences=cmdline.sequence,
geometry_restraints_manager=cmdline.geometry,
raw_data=cmdline.raw_data,
unmerged_data=cmdline.unmerged_i_obs,
all_chain_proxies=cmdline.processed_pdb_file.all_chain_proxies,
header_info=header_info,
keep_hydrogens=params.molprobity.keep_hydrogens,
nuclear=params.molprobity.nuclear,
save_probe_unformatted_file=probe_file,
min_cc_two_fofc=params.molprobity.min_cc_two_fofc,
n_bins_data=params.molprobity.n_bins,
outliers_only=params.molprobity.outliers_only,
use_pdb_header_resolution_cutoffs=\
params.molprobity.use_pdb_header_resolution_cutoffs,
count_anomalous_pairs_separately=\
params.molprobity.count_anomalous_pairs_separately,
use_internal_variance=params.input.unmerged_data.use_internal_variance,
file_name=params.input.pdb.file_name[0],
ligand_selection=params.molprobity.ligand_selection,
rotamer_library=params.molprobity.rotamer_library,
map_params=params)
map_file = None
if (not params.output.quiet) :
out2 = multi_out()
out2.register("stdout", out)
f = open(params.output.prefix + ".out", "w")
out2.register("txt_out", f)
validation.show(out=out2,
outliers_only=params.molprobity.outliers_only,
show_percentiles=params.output.percentiles)
f.close()
print >> out, ""
print >> out, "Results written to %s.out" % params.output.prefix
if (params.output.kinemage) :
if (cmdline.pdb_hierarchy.models_size() == 1) :
assert (probe_file is not None)
import mmtbx.kinemage.validation
kin_file = "%s.kin" % params.output.prefix
kin_out = \
mmtbx.kinemage.validation.export_molprobity_result_as_kinemage(
result=validation,
pdb_hierarchy=cmdline.pdb_hierarchy,
geometry=cmdline.geometry,
probe_file=probe_file,
keep_hydrogens=params.molprobity.keep_hydrogens,
pdbID=pdb_prefix)
f = open(kin_file, "w")
f.write(kin_out)
f.close()
if (not params.output.quiet) :
print >> out, "Wrote kinemage to %s" % kin_file
else :
print >> out, "Kinemage output not available for multiple MODELs."
if (params.output.pickle) :
easy_pickle.dump("%s.pkl" % params.output.prefix, validation)
if (not params.output.quiet) :
print >> out, "Saved result to %s.pkl" % params.output.prefix
if (params.output.coot) :
coot_file = "%s_coot.py" % params.output.prefix
validation.write_coot_script(coot_file)
if (not params.output.quiet) :
print >> out, "Wrote script for Coot: %s" % coot_file
if (params.output.maps == True) :
import mmtbx.maps.utils
import iotbx.map_tools
map_file = "%s_maps.mtz" % params.output.prefix
two_fofc_map, fofc_map = mmtbx.maps.utils.get_maps_from_fmodel(
fmodel=fmodel,
fill_missing_f_obs=params.output.map_options.fill_missing_f_obs,
exclude_free_r_reflections=\
params.output.map_options.exclude_free_r_reflections)
anom_map = None
if (fmodel.f_obs().anomalous_flag()) :
anom_map = mmtbx.maps.utils.get_anomalous_map(fmodel)
iotbx.map_tools.write_map_coeffs(
file_name=map_file,
fwt_coeffs=two_fofc_map,
delfwt_coeffs=fofc_map,
anom_coeffs=anom_map)
print >> out, "Wrote map coefficients to %s" % map_file
else :
print >> out, ""
validation.show_summary(out=out, show_percentiles=params.output.percentiles)
if (params.output.wxplots) :
try :
import wxtbx.app
except ImportError, e :
raise Sorry("wxPython not available.")
else :
app = wxtbx.app.CCTBXApp(0)
validation.display_wx_plots()
app.MainLoop()
def run(params, topdirs):
import sys
LogClass = dict(sp8=BssJobLog,
pf=None)
out_root = os.path.dirname(params.pklout)
out = multi_out()
out.register("log", open(os.path.join(out_root, "find_datasets.log"), "w"), atexit_send_to=None)
out.register("stdout", sys.stdout)
if params.find_spots:
shikalog = open(os.path.join(out_root, "shika.log"), "w")
shikalog.write("prefix idx nspots\n")
config_mgr = spot_finder_gui.ConfigManager(use_cuda=params.use_cuda)
logobjects = []
for topdir in topdirs:
print >>out, "Looking into %s\n" % topdir
for root, dirnames, filenames in os.walk(topdir, followlinks=True):
logs = filter(lambda x: x.endswith(".log"), filenames)
for log in logs:
log = os.path.join(root, log)
for logtype in params.logtype:
if 1:#try:
try:
logobj = LogClass[logtype](log)
except:
print >>out, traceback.format_exc()
print >>out, "\nException raised when parsing %s\n"%log
raise
logobj.jobs = filter(lambda x: x.job_mode != "XAFS", logobj.jobs)
if len(logobj.jobs) > 0:
print >>out, "Found job log:", log
spots = []
for job in logobj.jobs:
if params.find_spots:
spots.append(search_spots(os.path.join(root, os.path.basename(job.filename)),
job.n_images, config_mgr))
for idx, f, stat in spots[-1]:
if stat is None: continue
n_spots = stat.spots.get_n_spots("hi_pass_resolution_spots")
shikalog.write("%s %6d %4d\n" % (os.path.join(root,
os.path.basename(job.filename)),
idx, n_spots))
shikalog.flush()
else:
spots.append([])
logobjects.append([log, logobj, spots])
#except:
# pass
print >>out
logobjects.sort(key=lambda x:x[0])
for log, logobj, spots in logobjects:
print log
for job, spt in zip(logobj.jobs, spots):
jobtype = "?"
if job.advanced_centering == {} or test_gonio_coords_equal(job.advanced_centering.get("centers",[])):
jobtype = "Single "
elif job.advanced_centering.get("mode", "") == "vector_centering":
jobtype = "Helical "
elif job.advanced_centering.get("mode", "") == "multiple_centering":
jobtype = "MultiCen"
else:
print >>out, "WARNING:: WHY REACH HERE?", job.advanced_centering
osc_range = job.osc_end - job.osc_start
#nfound = check_files_exist(job.filename, job.n_images, os.path.dirname(log))
nfound = len(dataset.find_existing_files_in_template(job.filename, 1, job.n_images,
datadir=os.path.dirname(log),
check_compressed=True))
print >>out, " %s osc_step=%6.3f osc_range=%5.1f found=%d/%d %s beam_size=%s" % (job.beamline, job.osc_step, osc_range,
nfound, job.n_images, jobtype,
"x".join(map(lambda x:"%.1f"%x,job.beam_size)))
if params.find_spots:
n_spots = [stat.spots.get_n_spots("hi_pass_resolution_spots") for idx, f, stat in spt if stat is not None]
gt20 = len(filter(lambda x: x>=20, n_spots))
print >>out, " spots < 5A: %d..%d (>=20 spots: %d frames)" % (min(n_spots), max(n_spots), gt20)
print >>out
pickle.dump(logobjects, open(params.pklout, "w"), -1)
def run(args, command_name="phenix.explore_metric_symmetry"):
command_line = (
option_parser(
usage=command_name+" [options]",
description="""\
Explore Metric Symmetry. A list of possible unit cells and spacegroups is
given for the given specified unit cell and spacegroup combination. If a
second unit cell is given, linear combinations of the basis vector of one
unit cell are sought that match the other.""")
.enable_symmetry_comprehensive()
.option(None, "--max_delta",
action = "store",
type="float",
default=5.0,
dest = "max_delta",
help = "Maximum delta/obliquity used in determining the lattice symmetry, using a modified Le-Page algorithm. Default is 5.0 degrees",
metavar="FLOAT")
.option(None, "--start_from_p1",
action="store_true",
dest="niggli",
default=False,
help="Reduce to Niggli cell and forget the input spacegroup before higher metric symmetry is sought.")
.option(None, "--graph",
action="store",
default=None,
help="A graphical representation of the graph will be written out."
" Requires Graphviz to be installed and on PATH.")
.option(None, "--centring_type",
action="store",
type="str",
help="Centring type, choose from P,A,B,C,I,R,F")
.option(None, "--other_unit_cell",
action="store",
type="str",
help="Other unit cell, for unit cell comparison",
metavar="10,20,30,90,103.7,90")
.option(None, "--other_space_group",
action="store",
type="str",
help="space group for other_unit_cell, for unit cell comparison")
.option(None, "--other_centring_type",
action="store",
type="str",
help="Centring type, choose from P,A,B,C,I,R,F")
.option(None, "--no_point_group_graph",
action="store_true",
dest="pg_graph",
default=False,
help="Do not carry out the construction of a point group graph." )
.option(None, "--relative_length_tolerance",
action="store",
type="float",
help="Tolerance for unit cell lengths to be considered equal-ish.",
default=0.10,
metavar="FLOAT",
dest="rel_length_tol")
.option(None, "--absolute_angle_tolerance",
action="store",
dest="abs_angle_tol",
type="float",
default=10.0,
metavar="FLOAT",
help="Angular tolerance in unit cell comparison")
.option(None, "--max_order",
action="store",
type="int",
default=1,
metavar="INT",
help="Maximum volume change for target cell" )
).process(args=args)
log = multi_out()
log.register(label="stdout", file_object=sys.stdout)
allowed_centring_types={"P":"Primitive",
"A":"A centered",
"B":"B centered",
"C":"C centered",
"I":"Body centered",
"R":"Rombohedral",
"F":"Face centered"}
if command_line.options.centring_type is not None:
if not allowed_centring_types.has_key( command_line.options.centring_type ):
print >> log, "Sorry, the centring type %s is not known."%(command_line.options.centring_type)
print >> log, "Choose from P,A,B,C,I,R,F "
return
xs = None
other_xs = None
if len(args)==0:
command_line.parser.show_help()
return
if ( command_line.symmetry.unit_cell() == None ):
print >> log
print >> log, "Sorry: Unit cell not specified."
print >> log
command_line.parser.show_help()
return
if command_line.options.centring_type is None:
if ( command_line.symmetry.space_group_info() == None ):
print>> log
print>> log, "Sorry: centring type or space group not specified."
print>> log
command_line.parser.show_help()
return
if command_line.symmetry.space_group_info() is not None:
if not ( command_line.symmetry.space_group().is_chiral() ):
print >> log, "Sorry, Non chiral space groups not yet supported."
return
if command_line.options.centring_type is not None:
xs = crystal.symmetry(
unit_cell=command_line.symmetry.unit_cell(),
space_group_symbol="Hall: %s 1" %( command_line.options.centring_type )
)
command_line.symmetry = xs
if command_line.options.niggli:
print >> log, "*Unit cell will be niggli reduced and P1 will be assumed*"
uc = command_line.symmetry.change_basis(
command_line.symmetry.change_of_basis_op_to_niggli_cell() ).unit_cell()
command_line.symmetry = crystal.symmetry( uc, "P 1" )
xs = command_line.symmetry
############################################################################
# ABOVE IS JUST INPUT PARSING, NOW THE ACTUAL STUFF HAPPENS
############################################################################
if not command_line.options.pg_graph:
##############################
# get a point group graph #
##############################
pg_object = do_pointgroup_tricks( xs.unit_cell(),
xs.space_group(),
command_line.options.max_delta,
log )
################################################
# make a graphical representation if desired #
################################################
if command_line.options.graph is not None:
make_graph_of_graph(pg_object,
command_line.options.graph,
log)
#########################################
# Check if other cell has been defined #
#########################################
if command_line.options.other_unit_cell is not None:
print >> log, "A second unit cell has been specified. "
other_xs = None
if command_line.options.other_space_group is None:
if command_line.options.other_centring_type is None:
raise Sorry("No space group or centring type for other cell specified.")
else:
other_xs = crystal.symmetry( command_line.options.other_unit_cell,
space_group_symbol="Hall: %s 1" %( command_line.options.other_centring_type )
)
else:
other_xs = crystal.symmetry( command_line.options.other_unit_cell,
space_group_symbol=command_line.options.other_space_group
)
# get the graph is desired
if not command_line.options.pg_graph:
other_pg_object = do_pointgroup_tricks( other_xs.unit_cell(),
other_xs.space_group(),
command_line.options.max_delta,
log )
# do the unit cell comparison
print >> log
print >> log
print >> log, "Unit cell comparison"
print >> log, "--------------------"
print >> log
print >> log, "The unit cells will be compared. The smallest niggli cell,"
print >> log, "will be used as a (semi-flexible) lego-block to see if it"
print >> log, "can construct the larger Niggli cell."
print >> log
print >> log
order = command_line.options.max_order
if order==1:
sl_object = slt.compare_lattice(xs_a=xs,
xs_b=other_xs,
max_delta=command_line.options.max_delta,
out=log,
relative_length_tolerance=command_line.options.rel_length_tol,
absolute_angle_tolerance=command_line.options.abs_angle_tol)
else:
tmp_a = xs.change_basis( xs.change_of_basis_op_to_niggli_cell() )
tmp_b = other_xs.change_basis( other_xs.change_of_basis_op_to_niggli_cell() )
modified_xs = None
order = command_line.options.max_order
lego_block = None
if ( tmp_a.unit_cell().volume() > tmp_b.unit_cell().volume() ):
modified_xs = slt.make_list_of_target_xs_up_to_order( xs, order )
lego_block = other_xs
else:
modified_xs = slt.make_list_of_target_xs_up_to_order( other_xs, order )
lego_block = xs
print >> log
print >> log, "Volume change of largest niggli cell requested via keyword --max_order"
print >> log
print >> log, "Input crystal symmetry is tranformed to niggli setting using the operator:"
print >> log, modified_xs.basic_to_niggli_cb_op.as_xyz()
print >> log
print >> log, "Comparisons for various sublattices of the target cell are listed"
print >> log
for tmp_xs,cb_op,mat in zip(modified_xs.xs_list,
modified_xs.extra_cb_op,
modified_xs.matrices ):
mat=mat.as_list_of_lists()
print >> log, "==================================================================="
print >> log, "Niggli cell is expanded using matrix:"
print >> log
print >> log, " /%4i %4i %4i \ "%(mat[0][0],mat[0][1],mat[0][2])
print >> log, " M = |%4i %4i %4i | "%(mat[1][0],mat[1][1],mat[1][2])
print >> log, " \%4i %4i %4i / "%(mat[2][0],mat[2][1],mat[2][2])
print >> log
print >> log, "Change of basis operator to reference setting:"
print >> log, " ", cb_op.as_xyz()
print >> log, "resulting crystal symmetry:"
tmp_xs.show_summary(f=log,prefix=" ")
print >> log
print >> log
sl_object = slt.compare_lattice(xs_a=tmp_xs,
xs_b=lego_block,
max_delta=command_line.options.max_delta,
out=log,
relative_length_tolerance=command_line.options.rel_length_tol,
absolute_angle_tolerance=command_line.options.abs_angle_tol)
def __init__ (self,
args,
master_phil,
out=sys.stdout,
process_pdb_file=True,
require_data=True,
create_fmodel=True,
prefer_anomalous=None,
force_non_anomalous=False,
set_wavelength_from_model_header=False,
set_inelastic_form_factors=None,
usage_string=None,
create_log_buffer=False,
remove_unknown_scatterers=False,
generate_input_phil=False) :
import mmtbx.monomer_library.pdb_interpretation
import mmtbx.monomer_library.server
import mmtbx.utils
from iotbx import crystal_symmetry_from_any
from iotbx import file_reader
import iotbx.phil
if generate_input_phil :
assert isinstance(master_phil, basestring)
master_phil = generate_master_phil_with_inputs(phil_string=master_phil)
if isinstance(master_phil, str) :
master_phil = iotbx.phil.parse(master_phil)
if (usage_string is not None) :
if (len(args) == 0) or ("--help" in args) :
raise Usage("""%s\n\nFull parameters:\n%s""" % (usage_string,
master_phil.as_str(prefix=" ")))
if (force_non_anomalous) :
assert (not prefer_anomalous)
assert (set_inelastic_form_factors in [None, "sasaki", "henke"])
self.args = args
self.master_phil = master_phil
self.processed_pdb_file = self.pdb_inp = None
self.pdb_hierarchy = self.xray_structure = None
self.geometry = None
self.sequence = None
self.fmodel = None
self.f_obs = None
self.r_free_flags = None
self.intensity_flag = None
self.raw_data = None
self.raw_flags = None
self.test_flag_value = None
self.miller_arrays = None
self.hl_coeffs = None
self.cif_objects = []
self.log = out
if ("--quiet" in args) or ("quiet=True" in args) :
self.log = null_out()
elif create_log_buffer :
self.log = multi_out()
self.log.register(label="stdout", file_object=out)
self.log.register(label="log_buffer", file_object=StringIO())
make_header("Collecting inputs", out=self.log)
cmdline = iotbx.phil.process_command_line_with_files(
args=args,
master_phil=master_phil,
pdb_file_def="input.pdb.file_name",
reflection_file_def="input.xray_data.file_name",
cif_file_def="input.monomers.file_name",
seq_file_def="input.sequence")
self.working_phil = cmdline.work
params = self.working_phil.extract()
if len(params.input.pdb.file_name) == 0 :
raise Sorry("At least one PDB file is required as input.")
self.cif_file_names = params.input.monomers.file_name
self.pdb_file_names = params.input.pdb.file_name
# SYMMETRY HANDLING - PDB FILES
self.crystal_symmetry = pdb_symm = None
for pdb_file_name in params.input.pdb.file_name :
pdb_symm = crystal_symmetry_from_any.extract_from(pdb_file_name)
if (pdb_symm is not None) :
break
# DATA INPUT
data_and_flags = hkl_symm = hkl_in = None
if (params.input.xray_data.file_name is None) :
if (require_data) :
raise Sorry("At least one reflections file is required as input.")
else :
# FIXME this may still require that the data file has full crystal
# symmetry defined (although for MTZ input this will not be a problem)
make_sub_header("Processing X-ray data", out=self.log)
hkl_in = file_reader.any_file(params.input.xray_data.file_name)
hkl_in.check_file_type("hkl")
hkl_server = hkl_in.file_server
symm = hkl_server.miller_arrays[0].crystal_symmetry()
if ((symm is None) or
(symm.space_group() is None) or
(symm.unit_cell() is None)) :
if (pdb_symm is not None) :
from iotbx.reflection_file_utils import reflection_file_server
print >> self.log, \
"No symmetry in X-ray data file - using PDB symmetry:"
pdb_symm.show_summary(f=out, prefix=" ")
hkl_server = reflection_file_server(
crystal_symmetry=pdb_symm,
reflection_files=[hkl_in.file_object])
else :
raise Sorry("No crystal symmetry information found in input files.")
if (hkl_server is None) :
hkl_server = hkl_in.file_server
data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server=hkl_server,
parameters=params.input.xray_data,
data_parameter_scope="input.xray_data",
flags_parameter_scope="input.xray_data.r_free_flags",
prefer_anomalous=prefer_anomalous,
force_non_anomalous=force_non_anomalous,
log=self.log)
self.intensity_flag = data_and_flags.intensity_flag
self.raw_data = data_and_flags.raw_data
self.raw_flags = data_and_flags.raw_flags
self.test_flag_value = data_and_flags.test_flag_value
self.f_obs = data_and_flags.f_obs
self.r_free_flags = data_and_flags.r_free_flags
self.miller_arrays = hkl_in.file_server.miller_arrays
hkl_symm = self.raw_data.crystal_symmetry()
if len(self.cif_file_names) > 0 :
for file_name in self.cif_file_names :
cif_obj = mmtbx.monomer_library.server.read_cif(file_name=file_name)
self.cif_objects.append((file_name, cif_obj))
# SYMMETRY HANDLING - COMBINED
if (hkl_symm is not None) :
use_symmetry = hkl_symm
from iotbx.symmetry import combine_model_and_data_symmetry
self.crystal_symmetry = combine_model_and_data_symmetry(
model_symmetry=pdb_symm,
data_symmetry=hkl_symm)
if (self.crystal_symmetry is not None) and (self.f_obs is not None) :
self.f_obs = self.f_obs.customized_copy(
crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent().set_info(
self.f_obs.info())
self.r_free_flags = self.r_free_flags.customized_copy(
crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent().set_info(
self.r_free_flags.info())
# EXPERIMENTAL PHASES
target_name = "ml"
if hasattr(params.input, "experimental_phases") :
flag = params.input.use_experimental_phases
if (flag in [True, Auto]) :
phases_file = params.input.experimental_phases.file_name
if (phases_file is None) :
phases_file = params.input.xray_data.file_name
phases_in = hkl_in
else :
phases_in = file_reader.any_file(phases_file)
phases_in.check_file_type("hkl")
phases_in.file_server.err = self.log # redirect error output
space_group = self.crystal_symmetry.space_group()
point_group = space_group.build_derived_point_group()
hl_coeffs = mmtbx.utils.determine_experimental_phases(
reflection_file_server = phases_in.file_server,
parameters = params.input.experimental_phases,
log = self.log,
parameter_scope = "input.experimental_phases",
working_point_group = point_group,
symmetry_safety_check = True)
if (hl_coeffs is not None) :
hl_coeffs = hl_coeffs.map_to_asu()
if hl_coeffs.anomalous_flag() :
if (not self.f_obs.anomalous_flag()) :
hl_coeffs = hl_coeffs.average_bijvoet_mates()
elif self.f_obs.anomalous_flag() :
hl_coeffs = hl_coeffs.generate_bijvoet_mates()
self.hl_coeffs = hl_coeffs.matching_set(other=self.f_obs,
data_substitute=(0,0,0,0))
target_name = "mlhl"
# PDB INPUT
self.unknown_residues_flag = False
self.unknown_residues_error_message = False
if process_pdb_file :
pdb_interp_params = getattr(params, "pdb_interpretation", None)
if (pdb_interp_params is None) :
pdb_interp_params = \
mmtbx.monomer_library.pdb_interpretation.master_params.extract()
make_sub_header("Processing PDB file(s)", out=self.log)
pdb_combined = mmtbx.utils.combine_unique_pdb_files(
file_names=params.input.pdb.file_name,)
pdb_combined.report_non_unique(out=self.log)
pdb_raw_records = pdb_combined.raw_records
processed_pdb_files_srv = mmtbx.utils.process_pdb_file_srv(
cif_objects=self.cif_objects,
pdb_interpretation_params=pdb_interp_params,
crystal_symmetry=self.crystal_symmetry,
use_neutron_distances=params.input.scattering_table=="neutron",
stop_for_unknowns=getattr(pdb_interp_params, "stop_for_unknowns",False),
log=self.log)
self.processed_pdb_file, self.pdb_inp = \
processed_pdb_files_srv.process_pdb_files(
raw_records = pdb_raw_records,
stop_if_duplicate_labels = False,
allow_missing_symmetry=\
(self.crystal_symmetry is None) and (not require_data))
error_msg = self.processed_pdb_file.all_chain_proxies.\
fatal_problems_message(
ignore_unknown_scattering_types=False,
ignore_unknown_nonbonded_energy_types=False)
if (error_msg is not None) :
self.unknown_residues_flag = True
self.unknown_residues_error_message = error_msg
self.geometry = self.processed_pdb_file.geometry_restraints_manager(
show_energies=False)
assert (self.geometry is not None)
self.xray_structure = self.processed_pdb_file.xray_structure()
chain_proxies = self.processed_pdb_file.all_chain_proxies
self.pdb_hierarchy = chain_proxies.pdb_hierarchy
else :
pdb_file_object = mmtbx.utils.pdb_file(
pdb_file_names=params.input.pdb.file_name,
cif_objects=self.cif_objects,
crystal_symmetry=self.crystal_symmetry,
log=self.log)
self.pdb_inp = pdb_file_object.pdb_inp
self.pdb_hierarchy = self.pdb_inp.construct_hierarchy()
if (remove_unknown_scatterers) :
known_sel = self.pdb_hierarchy.atom_selection_cache().selection(
"not element X")
if (known_sel.count(True) != len(known_sel)) :
self.pdb_hierarchy = self.pdb_hierarchy.select(known_sel)
self.xray_structure = self.pdb_hierarchy.extract_xray_structure(
crystal_symmetry=self.crystal_symmetry)
self.pdb_hierarchy.atoms().reset_i_seq()
if (self.xray_structure is None) :
self.xray_structure = self.pdb_inp.xray_structure_simple(
crystal_symmetry=self.crystal_symmetry)
# wavelength
if (params.input.energy is not None) :
if (params.input.wavelength is not None) :
raise Sorry("Both wavelength and energy have been specified!")
params.input.wavelength = 12398.424468024265 / params.input.energy
if (set_wavelength_from_model_header and params.input.wavelength is None) :
wavelength = self.pdb_inp.extract_wavelength()
if (wavelength is not None) :
print >> self.log, ""
print >> self.log, "Using wavelength = %g from PDB header" % wavelength
params.input.wavelength = wavelength
# set scattering table
if (data_and_flags is not None) :
self.xray_structure.scattering_type_registry(
d_min=self.f_obs.d_min(),
table=params.input.scattering_table)
if ((params.input.wavelength is not None) and
(set_inelastic_form_factors is not None)) :
self.xray_structure.set_inelastic_form_factors(
photon=params.input.wavelength,
table=set_inelastic_form_factors)
make_sub_header("xray_structure summary", out=self.log)
self.xray_structure.scattering_type_registry().show(out = self.log)
self.xray_structure.show_summary(f=self.log)
# FMODEL SETUP
if (create_fmodel) and (data_and_flags is not None) :
make_sub_header("F(model) initialization", out=self.log)
skip_twin_detection = getattr(params.input, "skip_twin_detection", None)
twin_law = getattr(params.input, "twin_law", None)
if (twin_law is Auto) :
if (self.hl_coeffs is not None) :
raise Sorry("Automatic twin law determination not supported when "+
"experimental phases are used.")
elif (skip_twin_detection is not None) :
twin_law = Auto
if (twin_law is Auto) :
print >> self.log, "Twinning will be detected automatically."
self.fmodel = mmtbx.utils.fmodel_simple(
xray_structures=[self.xray_structure],
scattering_table=params.input.scattering_table,
f_obs=self.f_obs,
r_free_flags=self.r_free_flags,
skip_twin_detection=skip_twin_detection,
target_name=target_name,
log=self.log)
else :
if ((twin_law is not None) and (self.hl_coeffs is not None)) :
raise Sorry("Automatic twin law determination not supported when "+
"experimental phases are used.")
self.fmodel = mmtbx.utils.fmodel_manager(
f_obs=self.f_obs,
xray_structure=self.xray_structure,
r_free_flags=self.r_free_flags,
twin_law=params.input.twin_law,
hl_coeff=self.hl_coeffs,
target_name=target_name)
self.fmodel.update_all_scales(
params=None,
log=self.log,
optimize_mask=True,
show=True)
self.fmodel.info().show_rfactors_targets_scales_overall(out=self.log)
# SEQUENCE
if (params.input.sequence is not None) :
seq_file = file_reader.any_file(params.input.sequence,
force_type="seq",
raise_sorry_if_errors=True)
self.sequence = seq_file.file_object
# UNMERGED DATA
self.unmerged_i_obs = None
if hasattr(params.input, "unmerged_data") :
if (params.input.unmerged_data.file_name is not None) :
self.unmerged_i_obs = load_and_validate_unmerged_data(
f_obs=self.f_obs,
file_name=params.input.unmerged_data.file_name,
data_labels=params.input.unmerged_data.labels,
log=self.log)
self.params = params
print >> self.log, ""
print >> self.log, "End of input processing"
def simul_utils(args):
if len(args)==0:
print_help()
elif ( "--help" in args ):
print_help()
elif ( "--h" in args ):
print_help()
elif ("-h" in args ):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
print >> log, "#phil __OFF__"
print >> log, "======================"
print >> log, " SIMUL "
print >> log, "A data simulation tool"
print >> log, "======================"
print >> log
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
"""
new_params = master_params.format(python_object=params)
new_params.show(out=log)
"""
# now get the unit cell from the pdb file
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.xray_data.file_name)
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.simul_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group )
combined_xs = select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.simul_utils.input.unit_cell = combined_xs.unit_cell()
params.simul_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print >> log, "#phil __ON__"
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print >> log, "#phil __END__"
if params.simul_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.simul_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.simul_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.simul_utils.input.model.file_name is None:
raise Sorry("pdb file with model not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.simul_utils.input.unit_cell,
space_group_info=params.simul_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name = params.simul_utils.input.xray_data.file_name,
labels = params.simul_utils.input.xray_data.labels,
ignore_all_zeros = True,
parameter_scope = 'simul_utils.input.xray_data',
parameter_name = 'labels'
)
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(
miller_array.indices() != (0,0,0))
miller_array = miller_array.select(
miller_array.data() > 0 )
if miller_array.sigmas() is not None:
miller_array = miller_array.select(
miller_array.sigmas() > 0 )
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
print >> log
free_flags = miller_array.generate_r_free_flags()
#--------------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
model = pdb.input(file_name=params.simul_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs,
)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary()
print >> log
#-------------------------------------------------------------------
# Step 3: make an F_model object to get model phases and amplitudes
#
print >> log, "Performing bulk solvent scaling"
print >> log, "==============================="
print >> log
print >> log
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
f_model_object.update_solvent_and_scale(out=log)
fmodel = abs( f_model_object.f_model() ).set_observation_type( miller_array )
mockfmodel = None
if params.simul_utils.input.mock_model.file_name is not None:
print >> log, "Reading in mock model"
print >> log, "====================="
print >> log
print >> log
mock_model = pdb.input(file_name=params.simul_utils.input.mock_model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
mock_f_model = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = mock_model )
mock_f_model.update(
k_sol = f_model_object.k_sol() ,
b_sol = f_model_object.b_sol() ,
b_cart = f_model_object.b_cart()
)
mockfmodel = abs( mock_f_model.f_model() ).set_observation_type( miller_array )
else:
mockfmodel = fmodel.deep_copy()
print >> log, "Making new data"
print >> log, "==============="
print >> log
print >> log
new_data_builder = error_swap( miller_array,
fmodel,
mockfmodel )
new_data = new_data_builder.new_obs
# we now have to write the data actually
print >> log, "Writing new data set"
print >> log, "===================="
mtz_dataset = new_data.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset.mtz_object().write(
file_name=params.simul_utils.output.hklout)
def run(args):
if len(args) == 0:
master_params.show(expert_level=0)
elif ("--help" in args):
print("no help available")
elif ("--h" in args):
print("no help available")
elif ("--show_defaults" in args):
master_params.show(expert_level=0)
elif ("--show_defaults_all" in args):
master_params.show(expert_level=10)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
log_plots = StringIO()
print("#phil __OFF__", file=log)
print(file=log)
print(date_and_time(), file=log)
print(file=log)
print(file=log)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="scaling")
reflection_file = None
for arg in args:
command_line_params = None
arg_is_processed = False
if arg == '--quiet':
arg_is_processed = True
## The associated action with this keyword is implemented above
if (os.path.isfile(arg)): ## is this a file name?
## Check if this is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
except KeyboardInterrupt:
raise
except Exception:
pass
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
## Check if this file is a reflection file
if command_line_params is None:
reflection_file = reflection_file_reader.any_reflection_file(
file_name=arg, ensure_read_access=False)
if (reflection_file is not None):
reflection_file = arg
arg_is_processed = True
## If it is not a file, it must be a phil command
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print("##----------------------------------------------##",
file=log)
print("## Unknown phil-file or phil-command:", arg, file=log)
print("##----------------------------------------------##",
file=log)
print(file=log)
raise Sorry("Unknown file format or phil command: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
## Now please read in the reflections files
## get symmetry and cell data first please
## By default, the native cell and symmetry are used
## as reference
crystal_symmetry_nat = None
crystal_symmetry_nat = crystal_symmetry_from_any.extract_from(
file_name=params.scaling.input.xray_data.native.file_name)
if params.scaling.input.xray_data.space_group is None:
params.scaling.input.xray_data.space_group =\
crystal_symmetry_nat.space_group_info()
print("Using symmetry of native data", file=log)
if params.scaling.input.xray_data.unit_cell is None:
params.scaling.input.xray_data.unit_cell =\
crystal_symmetry_nat.unit_cell()
print("Using cell of native data", file=log)
## Check if a unit cell is defined
if params.scaling.input.xray_data.space_group is None:
raise Sorry("No space group defined")
if params.scaling.input.xray_data.unit_cell is None:
raise Sorry("No unit cell defined")
crystal_symmetry = crystal_symmetry = crystal.symmetry(
unit_cell=params.scaling.input.xray_data.unit_cell,
space_group_symbol=str(params.scaling.input.xray_data.space_group))
effective_params = master_params.fetch(sources=phil_objects)
new_params = master_params.format(python_object=params)
print("Effective parameters", file=log)
print("#phil __ON__", file=log)
new_params.show(out=log,
expert_level=params.scaling.input.expert_level)
print("#phil __END__", file=log)
print(file=log)
## define a xray data server
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=[])
## Read in native data and make appropriatre selections
miller_array_native = None
miller_array_native = xray_data_server.get_xray_data(
file_name=params.scaling.input.xray_data.native.file_name,
labels=params.scaling.input.xray_data.native.labels,
ignore_all_zeros=True,
parameter_scope='scaling.input.SIR_scale.xray_data.native')
info_native = miller_array_native.info()
miller_array_native = miller_array_native.map_to_asu().select(
miller_array_native.indices() != (0, 0, 0))
miller_array_native = miller_array_native.select(
miller_array_native.data() > 0)
## Convert to amplitudes
if (miller_array_native.is_xray_intensity_array()):
miller_array_native = miller_array_native.f_sq_as_f()
elif (miller_array_native.is_complex_array()):
miller_array_native = abs(miller_array_native)
if not miller_array_native.is_real_array():
raise Sorry("miller_array_native is not a real array")
miller_array_native.set_info(info=info_native)
## Read in derivative data and make appropriate selections
miller_array_derivative = None
miller_array_derivative = xray_data_server.get_xray_data(
file_name=params.scaling.input.xray_data.derivative.file_name,
labels=params.scaling.input.xray_data.derivative.labels,
ignore_all_zeros=True,
parameter_scope='scaling.input.SIR_scale.xray_data.derivative')
info_derivative = miller_array_derivative.info()
miller_array_derivative = miller_array_derivative.map_to_asu().select(
miller_array_derivative.indices() != (0, 0, 0))
miller_array_derivative = miller_array_derivative.select(
miller_array_derivative.data() > 0)
## Convert to amplitudes
if (miller_array_derivative.is_xray_intensity_array()):
miller_array_derivative = miller_array_derivative.f_sq_as_f()
elif (miller_array_derivative.is_complex_array()):
miller_array_derivative = abs(miller_array_derivative)
if not miller_array_derivative.is_real_array():
raise Sorry("miller_array_derivative is not a real array")
miller_array_derivative.set_info(info=info_derivative)
## As this is a SIR case, we will remove any anomalous pairs
if miller_array_derivative.anomalous_flag():
miller_array_derivative = miller_array_derivative.average_bijvoet_mates()\
.set_observation_type( miller_array_derivative )
if miller_array_native.anomalous_flag():
miller_array_native = miller_array_native.average_bijvoet_mates()\
.set_observation_type( miller_array_native )
## Print info
print(file=log)
print("Native data", file=log)
print("===========", file=log)
miller_array_native.show_comprehensive_summary(f=log)
print(file=log)
native_pre_scale = pre_scale.pre_scaler(
miller_array_native,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_native = native_pre_scale.x1.deep_copy()
del native_pre_scale
print(file=log)
print("Derivative data", file=log)
print("===============", file=log)
miller_array_derivative.show_comprehensive_summary(f=log)
print(file=log)
derivative_pre_scale = pre_scale.pre_scaler(
miller_array_derivative,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_derivative = derivative_pre_scale.x1.deep_copy()
del derivative_pre_scale
scaler = fa_estimation.combined_scaling(
miller_array_native, miller_array_derivative,
params.scaling.input.scaling_strategy.iso_protocol)
miller_array_native = scaler.x1.deep_copy()
miller_array_derivative = scaler.x2.deep_copy()
del scaler
print(file=log)
print("Making delta f's", file=log)
print("----------------", file=log)
print(file=log)
delta_gen = pair_analyses.delta_generator(miller_array_native,
miller_array_derivative)
print(file=log)
print("writing mtz file", file=log)
print("----------------", file=log)
print(file=log)
## some assertions to make sure nothing went weerd
assert miller_array_native.observation_type() is not None
assert miller_array_derivative.observation_type() is not None
assert delta_gen.abs_delta_f.observation_type() is not None
## Please write out the abs_delta_f array
mtz_dataset = delta_gen.abs_delta_f.as_mtz_dataset(
column_root_label='F' + params.scaling.input.output.outlabel)
mtz_dataset.mtz_object().write(
file_name=params.scaling.input.output.hklout)
def cmd_run(args, validated=False, out=sys.stdout):
if (len(args) == 0):
print >> out, "-"*79
print >> out, " phenix.polder"
print >> out, "-"*79
print >> out, legend
print >> out, "-"*79
master_params.show(out=out)
return
log = multi_out()
log.register("stdout", out)
log_file_name = "polder.log"
logfile = open(log_file_name, "w")
log.register("logfile", logfile)
print >> log, "phenix.polder is running..."
print >> log, "input parameters:\n", args
parsed = master_params
inputs = mmtbx.utils.process_command_line_args(args = args,
master_params = parsed)
#inputs.params.show() #check
params = inputs.params.extract()
# check model file
if len(inputs.pdb_file_names) == 0:
if (params.model_file_name is None):
raise Sorry("No model file found.")
elif (len(inputs.pdb_file_names) == 1):
params.model_file_name = inputs.pdb_file_names[0]
else:
raise Sorry("Only one model file should be given")
# check reflection file
reflection_files = inputs.reflection_files
if (len(reflection_files) == 0):
if (params.reflection_file_name is None):
raise Sorry("No reflection file found.")
else:
hkl_in = file_reader.any_file(params.reflection_file_name,
force_type="hkl")
hkl_in.assert_file_type("hkl")
reflection_files = [ hkl_in.file_object ]
# crystal symmetry
crystal_symmetry = None
crystal_symmetry = inputs.crystal_symmetry
if (crystal_symmetry is None):
crystal_symmetries = []
for f in [str(params.model_file_name), str(params.reflection_file_name)]:
cs = crystal_symmetry_from_any.extract_from(f)
if(cs is not None): crystal_symmetries.append(cs)
if(len(crystal_symmetries) == 1): crystal_symmetry = crystal_symmetries[0]
elif(len(crystal_symmetries) == 0):
raise Sorry("No crystal symmetry found.")
else:
if(not crystal_symmetries[0].is_similar_symmetry(crystal_symmetries[1])):
raise Sorry("Crystal symmetry mismatch between different files.")
crystal_symmetry = crystal_symmetries[0]
f_obs, r_free_flags = None, None
rfs = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files = reflection_files,
err = StringIO())
parameters = mmtbx.utils.data_and_flags_master_params().extract()
if (params.data_labels is not None):
parameters.labels = params.data_labels
if (params.r_free_flags_labels is not None):
parameters.r_free_flags.label = params.r_free_flags_labels
determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
reflection_file_server = rfs,
parameters = parameters,
keep_going = True,
log = StringIO())
f_obs = determined_data_and_flags.f_obs
if (params.data_labels is None):
params.data_labels = f_obs.info().label_string()
if (params.reflection_file_name is None):
params.reflection_file_name = parameters.file_name
r_free_flags = determined_data_and_flags.r_free_flags
assert f_obs is not None
print >> log, "Input data:"
print >> log, " Iobs or Fobs:", f_obs.info().labels
if (r_free_flags is not None):
print >> log, " Free-R flags:", r_free_flags.info().labels
params.r_free_flags_labels = r_free_flags.info().label_string()
else:
print >> log, " Free-R flags: Not present"
model_basename = os.path.basename(params.model_file_name.split(".")[0])
if (len(model_basename) > 0 and
params.output_file_name_prefix is None):
params.output_file_name_prefix = model_basename
print params.output_file_name_prefix
new_params = master_params.format(python_object=params)
new_params.show()
if (not validated):
validate_params(params)
pdb_input = iotbx.pdb.input(file_name = params.model_file_name)
pdb_hierarchy = pdb_input.construct_hierarchy()
xray_structure = pdb_hierarchy.extract_xray_structure(
crystal_symmetry = crystal_symmetry)
# DON'T USE:
# xray_structure = pdb_input.xray_structure_simple()
# atom order might be wrong
mmtbx.utils.setup_scattering_dictionaries(
scattering_table = params.scattering_table,
xray_structure = xray_structure,
d_min = f_obs.d_min())
#if f_obs is not None:
f_obs = f_obs.resolution_filter(
d_min = params.high_resolution,
d_max = params.low_resolution)
if (r_free_flags is not None):
r_free_flags = r_free_flags.resolution_filter(
d_min = params.high_resolution,
d_max = params.low_resolution)
# Grab case that data are anomalous
if (f_obs.anomalous_flag()):
f_obs, r_free_flags = prepare_f_obs_and_flags(
f_obs = f_obs,
r_free_flags = r_free_flags)
cpm_obj = compute_polder_map(
f_obs = f_obs,
r_free_flags = r_free_flags,
xray_structure = xray_structure,
pdb_hierarchy = pdb_hierarchy,
params = params,
log = log)
# Significance check
fmodel = mmtbx.f_model.manager(
f_obs = f_obs,
r_free_flags = r_free_flags,
xray_structure = xray_structure)
fmodel.update_all_scales(remove_outliers=False, fast=True)
f_obs_1 = abs(fmodel.f_model())
fmodel.update_xray_structure(xray_structure=cpm_obj.xray_structure_noligand,
update_f_calc=True, update_f_mask=True, force_update_f_mask=True)
# PVA: do we need it? fmodel.update_all_scales(remove_outliers=False)
f_obs_2 = abs(fmodel.f_model())
xrs_selected = cpm_obj.pdb_hierarchy_selected.extract_xray_structure(
crystal_symmetry = f_obs.crystal_symmetry())
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure = cpm_obj.xray_structure_noligand).f_calc()
f_mask = f_obs.structure_factors_from_map(
map = cpm_obj.mask_polder,
use_scale = True,
anomalous_flag = False,
use_sg = False)
def get_poler_diff_map(f_obs):
fmodel = mmtbx.f_model.manager(
f_obs = f_obs,
r_free_flags = r_free_flags,
f_calc = f_calc,
f_mask = f_mask)
fmodel.update_all_scales(remove_outliers=False)
mc_diff = map_tools.electron_density_map(
fmodel = fmodel).map_coefficients(
map_type = "mFo-DFc",
isotropize = True,
fill_missing = False)
fft_map = miller.fft_map(
crystal_gridding = cpm_obj.crystal_gridding,
fourier_coefficients = mc_diff)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
return mmtbx.utils.extract_box_around_model_and_map(
xray_structure = xrs_selected,
map_data = map_data,
box_cushion = 2.1)
box_1=get_poler_diff_map(f_obs = f_obs_1)
box_2=get_poler_diff_map(f_obs = f_obs_2)
box_3=get_poler_diff_map(f_obs = f_obs)
sites_cart_box = box_1.xray_structure_box.sites_cart()
sel = maptbx.grid_indices_around_sites(
unit_cell = box_1.xray_structure_box.unit_cell(),
fft_n_real = box_1.map_box.focus(),
fft_m_real = box_1.map_box.all(),
sites_cart = sites_cart_box,
site_radii = flex.double(sites_cart_box.size(), 2.0))
b1 = box_1.map_box.select(sel).as_1d()
b2 = box_2.map_box.select(sel).as_1d()
b3 = box_3.map_box.select(sel).as_1d()
print >> log, "Map 1: calculated Fobs with ligand"
print >> log, "Map 2: calculated Fobs without ligand"
print >> log, "Map 3: real Fobs data"
print >>log, "CC(1,2): %6.4f"%flex.linear_correlation(x=b1,y=b2).coefficient()
print >>log, "CC(1,3): %6.4f"%flex.linear_correlation(x=b1,y=b3).coefficient()
print >>log, "CC(2,3): %6.4f"%flex.linear_correlation(x=b2,y=b3).coefficient()
### D-function
b1 = maptbx.volume_scale_1d(map=b1, n_bins=10000).map_data()
b2 = maptbx.volume_scale_1d(map=b2, n_bins=10000).map_data()
b3 = maptbx.volume_scale_1d(map=b3, n_bins=10000).map_data()
print >> log, "Peak CC:"
print >>log, "CC(1,2): %6.4f"%flex.linear_correlation(x=b1,y=b2).coefficient()
print >>log, "CC(1,3): %6.4f"%flex.linear_correlation(x=b1,y=b3).coefficient()
print >>log, "CC(2,3): %6.4f"%flex.linear_correlation(x=b2,y=b3).coefficient()
cutoffs = flex.double(
[i/10. for i in range(1,10)]+[i/100 for i in range(91,100)])
d12 = maptbx.discrepancy_function(map_1=b1, map_2=b2, cutoffs=cutoffs)
d13 = maptbx.discrepancy_function(map_1=b1, map_2=b3, cutoffs=cutoffs)
d23 = maptbx.discrepancy_function(map_1=b2, map_2=b3, cutoffs=cutoffs)
print >> log, "q D(1,2) D(1,3) D(2,3)"
for c,d12_,d13_,d23_ in zip(cutoffs,d12,d13,d23):
print >> log, "%4.2f %6.4f %6.4f %6.4f"%(c,d12_,d13_,d23_)
###
if(params.debug):
box_1.write_ccp4_map(file_name="box_1_polder.ccp4")
box_2.write_ccp4_map(file_name="box_2_polder.ccp4")
box_3.write_ccp4_map(file_name="box_3_polder.ccp4")
cpm_obj.pdb_hierarchy_selected.adopt_xray_structure(
box_1.xray_structure_box)
cpm_obj.pdb_hierarchy_selected.write_pdb_file(file_name="box_polder.pdb",
crystal_symmetry=box_1.box_crystal_symmetry)
#
print >> log, "Finished."
return True
def find_merge_common_images(args):
phil = iotbx.phil.process_command_line(args = args,
master_string = master_phil).show()
work_params = phil.work.extract()
if ("--help" in args) :
libtbx.phil.parse(master_phil.show())
return
if ((work_params.d_min is None) or
(work_params.data is None) or
((work_params.model is None) and
work_params.scaling.algorithm != "mark1")) :
raise Usage("cxi.merge "
"d_min=4.0 "
"data=~/scratch/r0220/006/strong/ "
"model=3bz1_3bz2_core.pdb")
if ((work_params.rescale_with_average_cell) and
(not work_params.set_average_unit_cell)) :
raise Usage(
"If rescale_with_average_cell=True, you must also specify "+
"set_average_unit_cell=True.")
# Read Nat's reference model from an MTZ file. XXX The observation
# type is given as F, not I--should they be squared? Check with Nat!
log = open("%s_%s_scale.log" % (work_params.output.prefix,
work_params.scaling.algorithm), "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
print >> out, "Target unit cell and space group:"
print >> out, " ", work_params.target_unit_cell
print >> out, " ", work_params.target_space_group
uc = work_params.target_unit_cell
miller_set = symmetry(
unit_cell=work_params.target_unit_cell,
space_group_info=work_params.target_space_group
).build_miller_set(
anomalous_flag=not work_params.merge_anomalous,
d_min=work_params.d_min)
from xfel.cxi.merging.general_fcalc import random_structure
i_model = random_structure(work_params)
# ---- Augment this code with any special procedures for x scaling
scaler = xscaling_manager(
miller_set=miller_set,
i_model=i_model,
params=work_params,
log=out)
scaler.read_all()
print "finished reading"
sg = miller_set.space_group()
pg = sg.build_derived_laue_group()
miller_set.show_summary()
hkl_asu = scaler.observations["hkl_id"]
imageno = scaler.observations["frame_id"]
intensi = scaler.observations["i"]
sigma_i = scaler.observations["sigi"]
lookup = scaler.millers["merged_asu_hkl"]
# construct table of start / end indices for frames: now using Python
# range indexing
starts = [0]
ends = []
for x in xrange(1, len(scaler.observations["hkl_id"])):
if imageno[x] != imageno[x - 1]:
ends.append(x)
starts.append(x)
ends.append(len(scaler.observations["hkl_id"]))
keep_start = []
keep_end = []
for j, se in enumerate(zip(starts, ends)):
s, e = se
isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
dmin = 100.0
for x in xrange(s, e):
d = uc.d(lookup[hkl_asu[x]])
if d < dmin:
dmin = d
if isig > 6.0 and dmin < 3.2:
keep_start.append(s)
keep_end.append(e)
starts = keep_start
ends = keep_end
print 'Keeping %d frames' % len(starts)
frames = []
for s, e in zip(starts, ends):
indices = [tuple(lookup[hkl_asu[x]]) for x in range(s, e)]
intensities = intensi[s:e]
sigmas = sigma_i[s:e]
frames.append(Frame(uc, indices, intensities, sigmas))
from collections import defaultdict
hist = defaultdict(int)
fout = open('common_oh.dat', 'w')
for j, f in enumerate(frames):
hp = f.hand_pairs()
fout.write('%4d %d\n' % (j, hp))
hist[int(hp)] += 1
fout.close()
for b in sorted(hist):
print b, hist[b]
return
def run(args):
phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
work_params = phil.work.extract()
log = open("%s_%s_merging.log" % (work_params.output.prefix,work_params.scaling.algorithm), "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
print >> out, "Target unit cell and space group:"
print >> out, " ", work_params.target_unit_cell
print >> out, " ", work_params.target_space_group
miller_set = symmetry(
unit_cell=work_params.target_unit_cell,
space_group_info=work_params.target_space_group
).build_miller_set(
anomalous_flag=not work_params.merge_anomalous,
d_min=work_params.d_min)
from xfel.merging.general_fcalc import random_structure
i_model = random_structure(work_params)
# ---- Augment this code with any special procedures for x scaling
scaler = xscaling_manager(
miller_set=miller_set,
i_model=i_model,
params=work_params,
log=out)
scaler.read_all_mysql()
print "finished reading the database"
sg = miller_set.space_group()
hkl_asu = scaler.observations["hkl_id"]
imageno = scaler.observations["frame_id"]
intensi = scaler.observations["i"]
lookup = scaler.millers["merged_asu_hkl"]
origH = scaler.observations["H"]
origK = scaler.observations["K"]
origL = scaler.observations["L"]
from cctbx.array_family import flex
print "# observations from the database",len(scaler.observations["hkl_id"])
hkl = flex.miller_index(flex.select(lookup,hkl_asu))
from cctbx import miller
hkl_list = miller_set.customized_copy(indices = hkl)
ARRAY = miller.array(miller_set = hkl_list, data = intensi)
LATTICES = miller.array(miller_set = hkl_list, data = imageno)
from cctbx.merging.brehm_diederichs import run_multiprocess, run
L = (ARRAY, LATTICES) # tuple(data,lattice_id)
from libtbx import easy_pickle
presort_file = work_params.output.prefix+"_intensities_presort.pickle"
print "pickling these intensities to", presort_file
easy_pickle.dump(presort_file,L)
###### INPUTS #######
# data = miller array: ASU miller index + intensity (sigmas not implemented yet)
# lattice_id = flex double: assignment of each miller index to a lattice number
######################
if work_params.nproc < 5:
print "Sorting the lattices with 1 processor"
result = run(L,nproc=1,verbose=True)
else:
print "Sorting the lattices with %d processors"%work_params.nproc
result = run_multiprocess(L,nproc=work_params.nproc, verbose=False)
for key in result.keys():
print key,len(result[key])
# 2) pickle the postsort (reindexed) ARRAY, LATTICES XXX not done yet; not clear if needed
reverse_lookup = {}
frame_id_list = list(scaler.frames_mysql["frame_id"])
for key in result.keys():
for frame in result[key]:
frame_idx = frame_id_list.index(frame)
reverse_lookup[scaler.frames_mysql["unique_file_name"][frame_idx]] = key
lookup_file = work_params.output.prefix+"_lookup.pickle"
reverse_lookup_file = work_params.output.prefix+"_reverse_lookup.pickle"
easy_pickle.dump(lookup_file, result)
easy_pickle.dump(reverse_lookup_file, reverse_lookup)
def reindex_utils(args):
if len(args)==0:
print_help()
elif ( "--help" in args ):
print_help()
elif ( "--h" in args ):
print_help()
elif ("-h" in args ):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="map_coefs")
print >> log, "#phil __OFF__"
print >> log, "================="
print >> log, " REINDEX "
print >> log, "A reindexing tool"
print >> log, "================="
print >> log
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params_root = effective_params.extract()
params = params_root.reindex_utils
# now get the unit cell from the files
hkl_xs = None
pdb_xs = None
if params.input.xray_data.file_name is not None:
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.input.xray_data.file_name)
if params.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.input.unit_cell,
space_group_info=params.input.space_group )
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.input.unit_cell = combined_xs.unit_cell()
params.input.space_group = combined_xs.space_group_info()
print >> log, "#phil __ON__"
new_params = master_params.format(python_object=params_root)
new_params.show(out=log)
print >> log, "#phil __END__"
if params.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.input.space_group is None:
raise Sorry("space group not specified")
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
miller_array = None
if params.input.xray_data.file_name is not None:
phil_xs = crystal.symmetry(
unit_cell=params.input.unit_cell,
space_group_info=params.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = xray_data_server.get_xray_data(
file_name = params.input.xray_data.file_name,
labels = params.input.xray_data.labels,
ignore_all_zeros = True,
parameter_scope = 'reindex_utils.input.xray_data',
parameter_name = 'labels'
)
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(
miller_array.indices() != (0,0,0))
miller_array = miller_array.select(
miller_array.data() > 0 )
if miller_array.sigmas() is not None:
miller_array = miller_array.select(
miller_array.sigmas() > 0 )
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
print >> log
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
pdb_model = None
if params.input.model.file_name is not None:
pdb_model = iotbx.pdb.input(
file_name=params.input.model.file_name)
model = pdb_model.xray_structure_simple(crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary()
print >> log
if params.parameters.action=="reindex":
#----------------------------------------------------------------
# step 3: get the reindex laws
to_niggli = phil_xs.change_of_basis_op_to_niggli_cell()
to_reference = phil_xs.change_of_basis_op_to_reference_setting()
to_inverse = phil_xs.change_of_basis_op_to_inverse_hand()
cb_op = None
pr = params.parameters.reindex
if (pr.standard_laws == "niggli"):
cb_op = to_niggli
elif (pr.standard_laws == "reference_setting"):
cb_op = to_reference
elif (pr.standard_laws == "invert"):
cb_op = to_inverse
else:
cb_op = sgtbx.change_of_basis_op(pr.user_supplied_law)
if cb_op is None:
raise Sorry("No change of basis operation is supplied.")
if params.parameters.inverse:
cb_op = cb_op.inverse()
print >> log, "Supplied reindexing law:"
print >> log, "========================"
print >> log, "hkl notation: ", cb_op.as_hkl()
print >> log, "xyz notation: ", cb_op.as_xyz()
print >> log, "abc notation: ", cb_op.as_abc()
#----------------------------------------------------------------
# step 4: do the reindexing
#
# step 4a: first do the miller array object
new_miller_array = None
if miller_array is not None:
new_miller_array = miller_array.change_basis( cb_op )
#
# step 4b: the xray structure
new_model = None
if pdb_model is not None:
new_model = model.change_basis( cb_op )
#----------------------------------------------------------------
# step 5a: write the new mtz file
print >> log
print >> log, "The data and model have been reindexed"
print >> log, "--------------------------------------"
print >> log
print >> log, "Writing output files...."
if miller_array is not None:
print >> log, "writing mtz file with name %s" % (params.output.hklout)
mtz_dataset = new_miller_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset.mtz_object().write(file_name=params.output.hklout)
#step 5b: write the new pdb file
if new_model is not None:
pdb_file = open(params.output.xyzout, 'w')
print >> log, "Wring pdb file to: %s" % params.output.xyzout
write_as_pdb_file(
input_pdb = pdb_model,
input_xray_structure = new_model,
out = pdb_file,
chain_id_increment = params.parameters.chain_id_increment,
additional_remark = "Generated by mmtbx reindex")
print >> pdb_file, "END"
pdb_file.close()
if ( [miller_array,new_model]).count(None)==2:
print >>log, "No input reflection of coordinate files have been given"
if params.parameters.action=="operator":
rt_mx = sgtbx.rt_mx(
params.parameters.apply_operator.user_supplied_operator,t_den=12*8 )
if params.parameters.inverse:
rt_mx = rt_mx.inverse()
print >> log
print >> log, "Applied operator : ", rt_mx.as_xyz()
print >> log
sites = model.sites_frac()
new_sites = flex.vec3_double()
for site in sites:
new_site = rt_mx.r()*matrix.col(site)
new_site = flex.double(new_site)+flex.double( rt_mx.t().as_double() )
new_sites.append( tuple(new_site) )
new_model = model.deep_copy_scatterers()
new_model.set_sites_frac( new_sites )
# write the new [pdb file please
pdb_file = open(params.output.xyzout, 'w')
print >> log, "Wring pdb file to: %s" % params.output.xyzout
if params.parameters.apply_operator.concatenate_model:
write_as_pdb_file(
input_pdb = pdb_model,
input_xray_structure = model,
out = pdb_file,
chain_id_increment = 0,
additional_remark = None,
print_cryst_and_scale=True)
write_as_pdb_file(
input_pdb = pdb_model,
input_xray_structure = new_model,
out = pdb_file,
chain_id_increment = params.parameters.chain_id_increment,
additional_remark = None,
print_cryst_and_scale=False)
print >> pdb_file, "END"
pdb_file.close()
if params.parameters.action=="manipulate_pdb":
#rest all the b values
if params.parameters.manipulate_pdb.set_b:
b_iso = params.reindex_utils.parameters.manipulate_pdb.b_iso
new_model = model.set_b_iso( value = b_iso )
print >> log
print >> log, "All B-values have been set to %5.3f"%(b_iso)
print >> log, "Writing PDB file %s"%(params.output.xyzout)
print >> log
pdb_file = open(params.output.xyzout, 'w')
write_as_pdb_file(
input_pdb = pdb_model,
input_xray_structure = new_model,
out = pdb_file,
chain_id_increment = 0,
additional_remark = None,
print_cryst_and_scale=True)
print >> pdb_file, "END"
pdb_file.close()
#write the logfile
logger = open(params.output.logfile, 'w')
print >> log, "Writing log file with name %s" % params.output.logfile
print >> log
print >> logger, string_buffer.getvalue()
def find_merge_common_images(args):
phil = iotbx.phil.process_command_line(args = args,
master_string = master_phil).show()
work_params = phil.work.extract()
if ("--help" in args) :
libtbx.phil.parse(master_phil.show())
return
if ((work_params.d_min is None) or
(work_params.data is None) or
((work_params.model is None) and
work_params.scaling.algorithm != "mark1")) :
raise Usage("cxi.merge "
"d_min=4.0 "
"data=~/scratch/r0220/006/strong/ "
"model=3bz1_3bz2_core.pdb")
if ((work_params.rescale_with_average_cell) and
(not work_params.set_average_unit_cell)) :
raise Usage(
"If rescale_with_average_cell=True, you must also specify "+
"set_average_unit_cell=True.")
# Read Nat's reference model from an MTZ file. XXX The observation
# type is given as F, not I--should they be squared? Check with Nat!
log = open("%s_%s_scale.log" % (work_params.output.prefix,
work_params.scaling.algorithm), "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
print >> out, "Target unit cell and space group:"
print >> out, " ", work_params.target_unit_cell
print >> out, " ", work_params.target_space_group
uc = work_params.target_unit_cell
miller_set = symmetry(
unit_cell=work_params.target_unit_cell,
space_group_info=work_params.target_space_group
).build_miller_set(
anomalous_flag=not work_params.merge_anomalous,
d_min=work_params.d_min)
print 'Miller set size: %d' % len(miller_set.indices())
from xfel.cxi.merging.general_fcalc import random_structure
i_model = random_structure(work_params)
# ---- Augment this code with any special procedures for x scaling
scaler = xscaling_manager(
miller_set=miller_set,
i_model=i_model,
params=work_params,
log=out)
scaler.read_all()
print "finished reading"
sg = miller_set.space_group()
pg = sg.build_derived_laue_group()
miller_set.show_summary()
hkl_asu = scaler.observations["hkl_id"]
imageno = scaler.observations["frame_id"]
intensi = scaler.observations["i"]
sigma_i = scaler.observations["sigi"]
lookup = scaler.millers["merged_asu_hkl"]
# construct table of start / end indices for frames: now using Python
# range indexing
starts = [0]
ends = []
for x in xrange(1, len(scaler.observations["hkl_id"])):
if imageno[x] != imageno[x - 1]:
ends.append(x)
starts.append(x)
ends.append(len(scaler.observations["hkl_id"]))
keep_start = []
keep_end = []
def nint(a):
return int(round(a))
from collections import defaultdict
i_scale = 0.1
i_hist = defaultdict(int)
for j, se in enumerate(zip(starts, ends)):
s, e = se
for i in intensi[s:e]:
i_hist[nint(i_scale * i)] += 1
isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
dmin = 100.0
for x in xrange(s, e):
d = uc.d(lookup[hkl_asu[x]])
if d < dmin:
dmin = d
if isig > 6.0 and dmin < 3.2:
keep_start.append(s)
keep_end.append(e)
fout = open('i_hist.dat', 'w')
for i in i_hist:
fout.write('%.2f %d\n' % (i / i_scale, i_hist[i]))
fout.close()
starts = keep_start
ends = keep_end
print 'Keeping %d frames' % len(starts)
frames = []
odd = 0
even = 0
for s, e in zip(starts, ends):
for x in range(s, e):
hkl = lookup[hkl_asu[x]]
if (hkl[0] + hkl[1] + hkl[2]) % 2 == 1:
odd += 1
else:
even += 1
indices = [tuple(lookup[hkl_asu[x]]) for x in range(s, e)]
intensities = intensi[s:e]
sigmas = sigma_i[s:e]
frames.append(Frame(uc, indices, intensities, sigmas))
# pre-scale the data - first determine average ln(k), B; then apply
kbs = [f.kb() for f in frames]
mn_k = sum([kb[0] for kb in kbs]) / len(kbs)
mn_B = sum([kb[1] for kb in kbs]) / len(kbs)
n_lt_500 = 0
n_gt_500 = 0
for j, f in enumerate(frames):
s_i = f.scale_to_kb(mn_k, mn_B)
fout = open('frame-s-i-%05d.dat' % j, 'w')
for s, i, si in s_i:
fout.write('%f %f %f\n' % (s, i, si))
if i < 500:
n_lt_500 += 1
else:
n_gt_500 += 1
fout.close()
from collections import defaultdict
hist = defaultdict(int)
fout = open('kb.dat', 'w')
for j, f in enumerate(frames):
kb = f.kb()
fout.write('%4d %6.3f %6.3f\n' % (j, kb[0], kb[1]))
hist[int(round(kb[1]))] += 1
fout.close()
for b in sorted(hist):
print b, hist[b]
print odd, even
print n_lt_500, n_gt_500
return
def __init__ (self,
cmd_list,
nprocs=1,
out=sys.stdout,
log=None,
verbosity=DEFAULT_VERBOSITY,
output_junit_xml=False) :
if (log is None) : log = null_out()
self.out = multi_out()
self.log = log
self.out.register("stdout", out)
self.out.register("log", log)
self.verbosity = verbosity
self.quiet = (verbosity == 0)
self.results = []
# Filter cmd list for duplicates.
self.cmd_list = []
for cmd in cmd_list :
if (not cmd in self.cmd_list) :
self.cmd_list.append(cmd)
else :
print >> self.out, "Test %s repeated, skipping"%cmd
# Set number of processors.
if (nprocs is Auto) :
nprocs = cpu_count()
nprocs = min(nprocs, len(self.cmd_list))
# Starting summary.
if (self.verbosity > 0) :
print >> self.out, "Running %d tests on %s processors:"%(len(self.cmd_list), nprocs)
for cmd in self.cmd_list:
print >> self.out, " %s"%cmd
print >> self.out, ""
t_start = time.time()
if nprocs > 1:
# Run the tests with multiprocessing pool.
pool = Pool(processes=nprocs)
for command in self.cmd_list:
pool.apply_async(
run_command,
[command, verbosity, out],
callback=self.save_result)
try:
pool.close()
except KeyboardInterrupt:
print >> self.out, "Caught KeyboardInterrupt, terminating"
pool.terminate()
finally:
pool.join()
else:
# Run tests serially.
for command in self.cmd_list:
rc = run_command(command, verbosity=verbosity, out=out)
self.save_result(rc)
# Print ending summary.
t_end = time.time()
print >> self.out, "="*80
print >> self.out, ""
print >> self.out, "Tests finished. Elapsed time: %.2fs" %(t_end-t_start)
print >> self.out, ""
extra_stderr = 0
test_cases = []
# Process results for errors and warnings.
extra_stderr = len([result for result in self.results if result.stderr_lines])
longjobs = [result for result in self.results if result.wall_time > MAX_TIME]
warnings = [result for result in self.results if self.check_alert(result) == 1]
failures = [result for result in self.results if self.check_alert(result) == 2]
self.finished = len(self.results)
self.failure = len(failures)
self.warning = len(warnings)
# Output JUnit XML
if output_junit_xml:
from junit_xml import TestSuite, TestCase
for result in self.results:
tc = TestCase(name=result.command,
classname=result.command,
elapsed_sec=result.wall_time,
stdout='\n'.join(result.stdout_lines),
stderr='\n'.join(result.stderr_lines))
if result.return_code != 0:
tc.add_failure_info(message='exit code %d' %result.return_code)
#if len(result.stderr_lines):
#tc.add_error_info(output='\n'.join(result.stderr_lines))
test_cases.append(tc)
ts = TestSuite("libtbx.run_tests_parallel", test_cases=test_cases)
with open('output.xml', 'wb') as f:
print >> f, TestSuite.to_xml_string([ts], prettyprint=True)
# Run time distribution.
if (libtbx.env.has_module("scitbx")) :
from scitbx.array_family import flex
print >> self.out, "Distribution of test runtimes:"
hist = flex.histogram(flex.double([result.wall_time for result in self.results]), n_slots=10)
hist.show(f=self.out, prefix=" ", format_cutoffs="%.1fs")
print >> self.out, ""
# Long job warning.
if longjobs:
print >> self.out, ""
print >> self.out, "Warning: the following jobs took at least %d seconds:"%MAX_TIME
for result in sorted(longjobs, key=lambda result:result.wall_time):
print >> self.out, " %s: %.1fs"%(result.command, result.wall_time)
print >> self.out, "Please try to reduce overall runtime - consider splitting up these tests."
# Failures.
if failures:
print >> self.out, ""
print >> self.out, "Error: the following jobs returned non-zero exit codes or suspicious stderr output:"
print >> self.out, ""
for result in warnings:
self.display_result(result, alert=1, out=self.out, log_return=self.out, log_stderr=self.out)
for result in failures:
self.display_result(result, alert=2, out=self.out, log_return=self.out, log_stderr=self.out)
print >> self.out, ""
print >> self.out, "Please verify these tests manually."
print >> self.out, ""
# Summary
print >> self.out, "Summary:"
print >> self.out, " Tests run :",self.finished
print >> self.out, " Failures :",self.failure
print >> self.out, " Warnings (possible failures) :",self.warning
print >> self.out, " Stderr output (discouraged) :",extra_stderr
if (self.finished != len(self.cmd_list)) :
print >> self.out, "*" * 80
print >> self.out, " WARNING: NOT ALL TESTS FINISHED!"
print >> self.out, "*" * 80
def sfcalc(args):
if len(args) == 0:
print_help()
elif ("--help" in args):
print_help()
elif ("--h" in args):
print_help()
elif ("-h" in args):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="sfcalc")
print("#phil __OFF__", file=log)
print("=================", file=log)
print(" SFCALC ", file=log)
print("=================", file=log)
print(file=log)
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt:
raise
except Exception:
pass
if not arg_is_processed:
print("##----------------------------------------------##",
file=log)
print("## Unknown file or keyword:", arg, file=log)
print("##----------------------------------------------##",
file=log)
print(file=log)
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
# now get the unit cell from the files
hkl_xs = None
pdb_xs = None
if params.sfcalc.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.sfcalc.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.sfcalc.input.unit_cell,
space_group_info=params.sfcalc.input.space_group)
combined_xs = crystal.select_crystal_symmetry(None, phil_xs, [pdb_xs],
[None])
combined_xs.show_summary()
if combined_xs.unit_cell() is None:
raise Sorry("Unit cell not defined")
if combined_xs.space_group() is None:
raise Sorry("Space group not defined")
# inject the unit cell and symmetry in the phil scope please
params.sfcalc.input.unit_cell = combined_xs.unit_cell()
params.sfcalc.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print("#phil __ON__", file=log)
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print("#phil __END__", file=log)
pdb_model = None
if params.sfcalc.input.model.file_name is not None:
pdb_model = pdb.input(
file_name=params.sfcalc.input.model.file_name)
model = pdb_model.xray_structure_simple(crystal_symmetry=phil_xs)
print("Atomic model summary", file=log)
print("====================", file=log)
model.show_summary()
print(file=log)
#make an f_model object please
b_cart = params.sfcalc.parameters.overall.b_cart
b_cart = [
b_cart.b_11, b_cart.b_22, b_cart.b_33, b_cart.b_12,
b_cart.b_13, b_cart.b_23
]
dummy = abs(
model.structure_factors(d_min=params.sfcalc.parameters.d_min,
anomalous_flag=False).f_calc())
flags = dummy.generate_r_free_flags(fraction=0.1,
max_free=99999999)
fmodel = f_model.manager(
xray_structure=model,
r_free_flags=flags,
target_name="ls_wunit_k1",
f_obs=dummy,
b_cart=b_cart,
k_sol=params.sfcalc.parameters.solvent.k_sol,
b_sol=params.sfcalc.parameters.solvent.b_sol)
calc_data_with_solvent_contrib = fmodel.f_model()
calc_data_with_solvent_contrib = calc_data_with_solvent_contrib.array(
data=calc_data_with_solvent_contrib.data() *
params.sfcalc.parameters.overall.k_overall)
result = None
label = None
if params.sfcalc.parameters.output_type == "complex":
result = calc_data_with_solvent_contrib
label = "FMODEL"
if params.sfcalc.parameters.output_type == "amplitudes":
result = abs(calc_data_with_solvent_contrib)
label = "FMODEL"
if params.sfcalc.parameters.output_type == "intensities":
result = abs(calc_data_with_solvent_contrib)
result = result.f_as_f_sq()
label = "IMODEL"
#write an mtz file with the data
mtz_dataset = result.as_mtz_dataset(column_root_label=label)
mtz_dataset.mtz_object().write(
file_name=params.sfcalc.output.hklout)
#write the logfile
logger = open(params.sfcalc.output.logfile, 'w')
print("writing log file with name %s" % (params.sfcalc.output.logfile),
file=log)
print(file=log)
print(string_buffer.getvalue(), file=logger)
def __init__ (self,
cmd_list,
nprocs=1,
out=sys.stdout,
log=None,
quiet=False,
output_junit_xml=False) :
if (log is None) : log = null_out()
self.out = multi_out()
self.log = log
self.out.register("stdout", out)
self.out.register("log", log)
self.quiet = quiet
self.cmd_list = []
for cmd in cmd_list :
if (not cmd in self.cmd_list) :
self.cmd_list.append(cmd)
else :
print >> self.out, " test %s repeated, skipping" % cmd
nprocs = min(nprocs, len(self.cmd_list))
print >> self.out, "\n Starting command list"
print >> self.out, " NProcs :",nprocs
print >> self.out, " Cmds :",len(self.cmd_list)
t_start = time.time()
if nprocs>1:
pool = Pool(processes=nprocs)
self.results = []
for command in self.cmd_list:
if nprocs>1:
pool.apply_async(
run_command,
[command, (not quiet), out],
callback=self.save_result)
else:
rc = run_command(command, verbose=(not quiet), out=out)
self.save_result(rc)
if nprocs>1:
try :
try :
pool.close()
except KeyboardInterrupt :
print >> self.out, "Caught KeyboardInterrupt, terminating"
pool.terminate()
finally :
pool.join()
print >> self.out, '\nProcesses have joined : %d\n' % len(self.results)
t_end = time.time()
print >> self.out, ""
print >> self.out, "Elapsed time: %.2fs" %(t_end-t_start)
print >> self.out, ""
finished = 0
warning = 0
extra_stderr = 0
failure = 0
failures = []
long_jobs = []
long_runtimes = []
runtimes = []
if output_junit_xml:
from junit_xml import TestSuite, TestCase
test_cases = []
for result in self.results :
finished += 1
runtimes.append(result.wall_time)
if (result.return_code != 0) :
failure += 1
failures.append(result)
else :
if (len(result.error_lines) != 0) :
warning += 1
failures.append(result)
if (len(result.stderr_lines) != 0):
extra_stderr += 1
if (result.wall_time > max_time) :
long_jobs.append(result.command)
long_runtimes.append(result.wall_time)
if output_junit_xml:
tc = TestCase(name=result.command,
classname=result.command,
elapsed_sec=result.wall_time,
stdout='\n'.join(result.stdout_lines),
stderr='\n'.join(result.stderr_lines))
if result.return_code != 0:
tc.add_failure_info(message='exit code %d' %result.return_code)
#if len(result.stderr_lines):
#tc.add_error_info(output='\n'.join(result.stderr_lines))
test_cases.append(tc)
if output_junit_xml:
ts = TestSuite("libtbx.run_tests_parallel", test_cases=test_cases)
with open('output.xml', 'wb') as f:
print >> f, TestSuite.to_xml_string([ts], prettyprint=True)
if (libtbx.env.has_module("scitbx")) :
from scitbx.array_family import flex
print >> self.out, "Distribution of test runtimes:"
hist = flex.histogram(flex.double(runtimes), n_slots=20)
hist.show(f=self.out, prefix=" ", format_cutoffs="%.1fs")
print >> self.out, ""
if (len(long_jobs) > 0) :
print >> self.out, ""
print >> self.out, "WARNING: the following jobs took at least %d seconds:" % \
max_time
jobs_and_timings = list(zip(long_jobs, long_runtimes))
jobs_and_timings.sort(lambda x,y: cmp(x[1], y[1]))
for cmd, runtime in jobs_and_timings :
print >> self.out, " " + cmd + " : %.1fs" % runtime
print >> self.out, "Please try to reduce overall runtime - consider splitting up these tests."
if (len(failures) > 0) :
print >> self.out, ""
print >> self.out, "ERROR: the following jobs returned non-zero exit codes or suspicious stderr output:"
for result in failures :
print >> self.out, ""
print >> self.out, result.command + "(exit code %d):" % result.return_code
for line in result.stderr_lines :
print >> self.out, " " + line
for line in result.error_lines :
print >> self.out, " " + line
print >> self.out, ""
print >> self.out, "Please verify these tests manually."
print >> self.out, ""
print >> self.out, "Summary:"
print >> self.out, " Tests run :",finished
print >> self.out, " Failures :",failure
print >> self.out, " Warnings (possible failures) :",warning
print >> self.out, " Stderr output (discouraged) :",extra_stderr
if (finished != len(self.cmd_list)) :
print >> self.out, "*" * 80
print >> self.out, " WARNING: NOT ALL TESTS FINISHED!"
print >> self.out, "*" * 80
def __init__(self,
cmd_list,
nprocs=1,
out=sys.stdout,
log=None,
verbosity=DEFAULT_VERBOSITY,
max_time=180):
if (log is None): log = null_out()
self.out = multi_out()
self.log = log
self.out.register("stdout", out)
self.out.register("log", log)
self.verbosity = verbosity
self.quiet = (verbosity == 0)
self.results = []
# Filter cmd list for duplicates.
self.cmd_list = []
for cmd in cmd_list:
if (not cmd in self.cmd_list):
self.cmd_list.append(cmd)
else:
print >> self.out, "Test %s repeated, skipping" % cmd
# Set number of processors.
if (nprocs is Auto):
nprocs = cpu_count()
nprocs = min(nprocs, len(self.cmd_list))
# Starting summary.
if (self.verbosity > 0):
print >> self.out, "Running %d tests on %s processors:" % (len(
self.cmd_list), nprocs)
for cmd in self.cmd_list:
print >> self.out, " %s" % cmd
print >> self.out, ""
t_start = time.time()
if nprocs > 1:
# Run the tests with multiprocessing pool.
pool = Pool(processes=nprocs)
for command in self.cmd_list:
pool.apply_async(run_command, [command, verbosity, out],
callback=self.save_result)
try:
pool.close()
except KeyboardInterrupt:
print >> self.out, "Caught KeyboardInterrupt, terminating"
pool.terminate()
finally:
pool.join()
else:
# Run tests serially.
for command in self.cmd_list:
rc = run_command(command, verbosity=verbosity, out=out)
self.save_result(rc)
# Print ending summary.
t_end = time.time()
print >> self.out, "=" * 80
print >> self.out, ""
print >> self.out, "Tests finished. Elapsed time: %.2fs" % (t_end -
t_start)
print >> self.out, ""
test_cases = []
# Process results for errors and warnings.
extra_stderr = len(
[result for result in self.results if result.stderr_lines])
longjobs = [
result for result in self.results if result.wall_time > max_time
]
warnings = [
result for result in self.results if self.check_alert(result) == 1
]
failures = [
result for result in self.results if self.check_alert(result) == 2
]
self.finished = len(self.results)
self.failure = len(failures)
self.warning = len(warnings)
# Output JUnit XML if possible
try:
from junit_xml import TestSuite, TestCase
import re
def decode_string(string):
try:
return string.encode('ascii', 'xmlcharrefreplace')
except Exception: # intentional
return unicode(string, errors='ignore').encode(
'ascii', 'xmlcharrefreplace')
for result in self.results:
test_name = reconstruct_test_name(result.command)
plain_stdout = map(decode_string, result.stdout_lines)
plain_stderr = map(decode_string, result.stderr_lines)
output = '\n'.join(plain_stdout + plain_stderr)
tc = TestCase(classname=test_name[0],
name=test_name[1],
elapsed_sec=result.wall_time,
stdout='\n'.join(plain_stdout),
stderr='\n'.join(plain_stderr))
if result.return_code == 0:
# Identify skipped tests
if re.search('skip', output, re.IGNORECASE):
# find first line including word 'skip' and use it as message
skipline = re.search('^((.*)skip(.*))$', output,
re.IGNORECASE
| re.MULTILINE).group(1)
tc.add_skipped_info(skipline)
else:
# Test failed. Extract error message and stack trace if possible
error_message = 'exit code %d' % result.return_code
error_output = '\n'.join(plain_stderr)
if plain_stderr:
error_message = plain_stderr[-1]
if len(plain_stderr) > 20:
error_output = '\n'.join(plain_stderr[-20:])
tc.add_failure_info(message=error_message,
output=error_output)
test_cases.append(tc)
ts = TestSuite("libtbx.run_tests_parallel", test_cases=test_cases)
with open('output.xml', 'wb') as f:
print >> f, TestSuite.to_xml_string([ts], prettyprint=True)
except ImportError, e:
pass
def run(params):
if os.path.isdir(params.workdir) and os.listdir(params.workdir):
print "Directory already exists and not empty:", params.workdir
return
if params.reference_file is not None and params.program != "xscale":
print "WARNING - reference file is not used unless program=xscale."
if not os.path.isdir(params.workdir):
os.makedirs(params.workdir)
if params.batch.engine == "sge":
batchjobs = batchjob.SGE(pe_name=params.batch.sge_pe_name)
elif params.batch.engine == "sh":
batchjobs = batchjob.ExecLocal()
else:
raise "Unknown batch engine: %s" % params.batch.engine
out = multi_out()
out.register("log", open(os.path.join(params.workdir, "multi_merge.log"), "w"), atexit_send_to=None)
out.register("stdout", sys.stdout)
print >>out, "Paramters:"
libtbx.phil.parse(master_params_str).format(params).show(out=out, prefix=" ")
print >>out, ""
# XXX Not works when clustering is used..
html_report = multi_merging.html_report.HtmlReportMulti(os.path.abspath(params.workdir))
try: html_report.add_params(params, master_params_str)
except: print >>out, traceback.format_exc()
xds_ascii_files = map(lambda x: x[:(x.index("#") if "#" in x else None)].strip(), open(params.lstin))
xds_ascii_files = filter(lambda x: x!="" and os.path.isfile(x), xds_ascii_files)
xds_ascii_files = map(lambda x: os.path.abspath(x), xds_ascii_files)
cells = collections.OrderedDict()
laues = {} # for check
for xac in xds_ascii_files:
try:
symm = XDS_ASCII(xac, read_data=False).symm
except:
print >>out, "Error in reading %s" % xac
print >>out, traceback.format_exc()
return
cells[xac] = symm.unit_cell().parameters()
laue = symm.space_group().build_derived_reflection_intensity_group(False).info()
laues.setdefault(str(laue),{}).setdefault(symm.space_group_info().type().number(), []).append(xac)
if len(laues) > 1:
print >>out, "ERROR! more than one space group included."
for laue in laues:
print "Laue symmetry", laue
for sg in laues[laue]:
print >>out, " SPACE_GROUP_NUMBER= %d (%d data)" % (sg, len(laues[laue][sg]))
for f in laues[laue][sg]: print >>out, " %s" % f
print >>out, ""
return
try: html_report.add_cells_and_files(cells, laues.keys()[0])
except: print >>out, traceback.format_exc()
data_for_merge = []
if params.clustering == "blend":
if params.blend.use_old_result is None:
blend_wdir = os.path.join(params.workdir, "blend")
os.mkdir(blend_wdir)
blend.run_blend0R(blend_wdir, xds_ascii_files)
print >>out, "\nRunning BLEND with analysis mode"
else:
blend_wdir = params.blend.use_old_result
print >>out, "\nUsing precalculated BLEND result in %s" % params.blend.use_old_result
blend_clusters = blend.BlendClusters(workdir=blend_wdir, d_min=params.d_min)
summary_out = os.path.join(blend_wdir, "blend_cluster_summary.dat")
clusters = blend_clusters.show_cluster_summary(out=open(summary_out, "w"))
print >>out, "Clusters found by BLEND were summarized in %s" % summary_out
if params.blend.min_cmpl is not None:
clusters = filter(lambda x: x[3] >= params.blend.min_cmpl, clusters)
if params.blend.min_acmpl is not None:
clusters = filter(lambda x: x[5] >= params.blend.min_acmpl, clusters)
if params.blend.min_redun is not None:
clusters = filter(lambda x: x[4] >= params.blend.min_redun, clusters)
if params.blend.min_aredun is not None:
clusters = filter(lambda x: x[6] >= params.blend.min_aredun, clusters)
if params.blend.max_LCV is not None:
clusters = filter(lambda x: x[7] <= params.blend.max_LCV, clusters)
if params.blend.max_aLCV is not None:
clusters = filter(lambda x: x[8] <= params.blend.max_aLCV, clusters)
print >>out, "With specified conditions, following %d clusters will be merged:" % len(clusters)
for clno, IDs, clh, cmpl, redun, acmpl, aredun, LCV, aLCV in clusters: # process largest first
print >>out, " Cluster_%.4d NumDS= %4d CLh= %5.1f Cmpl= %6.2f Redun= %4.1f ACmpl=%6.2f ARedun=%4.1f LCV= %5.1f aLCV=%5.1f" % (clno, len(IDs), clh, cmpl, redun, acmpl, aredun, LCV, aLCV)
data_for_merge.append((os.path.join(params.workdir, "cluster_%.4d"%clno),
map(lambda x: blend_clusters.files[x-1], IDs),
LCV, aLCV,clh))
print >>out
try: html_report.add_clutering_result(clusters, "blend")
except: print >>out, traceback.format_exc()
elif params.clustering == "cc":
ccc_wdir = os.path.join(params.workdir, "cc_clustering")
os.mkdir(ccc_wdir)
cc_clusters = cc_clustering.CCClustering(ccc_wdir, xds_ascii_files,
d_min=params.cc_clustering.d_min,
min_ios=params.cc_clustering.min_ios)
print >>out, "\nRunning CC-based clustering"
cc_clusters.do_clustering(nproc=params.cc_clustering.nproc,
b_scale=params.cc_clustering.b_scale,
use_normalized=params.cc_clustering.use_normalized,
html_maker=html_report)
summary_out = os.path.join(ccc_wdir, "cc_cluster_summary.dat")
clusters = cc_clusters.show_cluster_summary(d_min=params.d_min, out=open(summary_out, "w"))
print >>out, "Clusters were summarized in %s" % summary_out
if params.cc_clustering.min_cmpl is not None:
clusters = filter(lambda x: x[3] >= params.cc_clustering.min_cmpl, clusters)
if params.cc_clustering.min_acmpl is not None:
clusters = filter(lambda x: x[5] >= params.cc_clustering.min_acmpl, clusters)
if params.cc_clustering.min_redun is not None:
clusters = filter(lambda x: x[4] >= params.cc_clustering.min_redun, clusters)
if params.cc_clustering.min_aredun is not None:
clusters = filter(lambda x: x[6] >= params.cc_clustering.min_aredun, clusters)
if params.cc_clustering.max_clheight is not None:
clusters = filter(lambda x: x[2] <= params.cc_clustering.max_clheight, clusters)
print >>out, "With specified conditions, following %d clusters will be merged:" % len(clusters)
for clno, IDs, clh, cmpl, redun, acmpl, aredun in clusters: # process largest first
print >>out, " Cluster_%.4d NumDS= %4d CLh= %5.1f Cmpl= %6.2f Redun= %4.1f ACmpl=%6.2f ARedun=%4.1f" % (clno, len(IDs), clh, cmpl, redun, acmpl, aredun)
data_for_merge.append((os.path.join(params.workdir, "cluster_%.4d"%clno),
map(lambda x: xds_ascii_files[x-1], IDs),
float("nan"),float("nan"),clh))
print >>out
try: html_report.add_clutering_result(clusters, "cc_clustering")
except: print >>out, traceback.format_exc()
else:
data_for_merge.append((os.path.join(params.workdir, "all_data"),
xds_ascii_files, float("nan"), float("nan"), 0))
ofs_summary = open(os.path.join(params.workdir, "cluster_summary.dat"), "w")
ofs_summary.write("# d_min= %.3f A\n" % (params.d_min if params.d_min is not None else float("nan")))
ofs_summary.write("# LCV and aLCV are values of all data\n")
ofs_summary.write(" cluster ClH LCV aLCV run ds.all ds.used Cmpl Redun I/sigI Rmeas CC1/2 Cmpl.ou Red.ou I/sig.ou Rmeas.ou CC1/2.ou Cmpl.in Red.in I/sig.in Rmeas.in CC1/2.in SigAno.in CCano.in WilsonB Aniso \n")
out.flush()
def write_ofs_summary(workdir, cycle, clh, LCV, aLCV, xds_files, num_files, stats):
tmps = "%12s %5.2f %4.1f %4.1f %3d %6d %7d %5.1f %5.1f %6.2f %5.1f %5.1f %7.1f %6.1f % 8.2f % 8.1f %8.1f %7.1f %6.1f % 8.2f % 8.1f %8.1f %9.1f %8.1f %7.2f %7.1e\n"
ofs_summary.write(tmps % (os.path.relpath(workdir, params.workdir), clh, LCV, aLCV, cycle,
len(xds_files), num_files,
stats["cmpl"][0],
stats["redundancy"][0],
stats["i_over_sigma"][0],
stats["r_meas"][0],
stats["cc_half"][0],
stats["cmpl"][2],
stats["redundancy"][2],
stats["i_over_sigma"][2],
stats["r_meas"][2],
stats["cc_half"][2],
stats["cmpl"][1],
stats["redundancy"][1],
stats["i_over_sigma"][1],
stats["r_meas"][1],
stats["cc_half"][1],
stats["sig_ano"][1],
stats["cc_ano"][1],
stats["xtriage_log"].wilson_b,
stats["xtriage_log"].anisotropy,
))
ofs_summary.flush()
# write_ofs_summary()
if "merging" in params.batch.par_run:
params.nproc = params.batch.nproc_each
jobs = []
for workdir, xds_files, LCV, aLCV, clh in data_for_merge:
if not os.path.exists(workdir): os.makedirs(workdir)
shname = "merge_%s.sh" % os.path.relpath(workdir, params.workdir)
pickle.dump((params, os.path.abspath(workdir), xds_files, cells, batchjobs), open(os.path.join(workdir, "args.pkl"), "w"), -1)
job = batchjob.Job(workdir, shname, nproc=params.batch.nproc_each)
job.write_script("""\
"%s" -c '\
import pickle; \
from yamtbx.dataproc.auto.command_line.multi_merge import merge_datasets; \
args = pickle.load(open("args.pkl")); \
ret = merge_datasets(*args); \
pickle.dump(ret, open("result.pkl","w")); \
'
""" % sys.executable)
batchjobs.submit(job)
jobs.append(job)
batchjobs.wait_all(jobs)
for workdir, xds_files, LCV, aLCV, clh in data_for_merge:
try:
results = pickle.load(open(os.path.join(workdir, "result.pkl")))
except:
print >>out, "Error in unpickling result in %s" % workdir
print >>out, traceback.format_exc()
results = []
if len(results) == 0:
ofs_summary.write("#%s failed\n" % os.path.relpath(workdir, params.workdir))
for cycle, wd, num_files, stats in results:
write_ofs_summary(workdir, cycle, clh, LCV, aLCV, xds_files, num_files, stats)
try: html_report.add_merge_result(workdir, clh, LCV, aLCV, xds_files, results[-1][2], results[-1][3])
except: print >>out, traceback.format_exc()
else:
for workdir, xds_files, LCV, aLCV, clh in data_for_merge:
print >>out, "Merging %s..." % os.path.relpath(workdir, params.workdir)
out.flush()
results = merge_datasets(params, workdir, xds_files, cells, batchjobs)
if len(results) == 0:
ofs_summary.write("#%s failed\n" % os.path.relpath(workdir, params.workdir))
for cycle, wd, num_files, stats in results:
write_ofs_summary(workdir, cycle, clh, LCV, aLCV, xds_files, num_files, stats)
try: html_report.add_merge_result(workdir, clh, LCV, aLCV, xds_files, results[-1][2], results[-1][3])
except: print >>out, traceback.format_exc()
try: html_report.write_html()
except: print >>out, traceback.format_exc()
print "firefox %s" % os.path.join(html_report.root, "report.html")
return
def run (args,
out=sys.stdout,
program_name="phenix.molprobity",
ignore_missing_modules=False,
return_input_objects=False) : # for testing
rotarama_dir = libtbx.env.find_in_repositories(
relative_path="chem_data/rotarama_data",
test=os.path.isdir)
if (rotarama_dir is None) :
raise ImportError("Rotamer and Ramachandran distributions not available; "+
"you will need these to run MolProbity.")
elif (((not libtbx.env.has_module("reduce")) or
(not libtbx.env.has_module("probe"))) and
(not ignore_missing_modules)) :
raise ImportError("Reduce and/or Probe not configured.")
import mmtbx.validation.molprobity
import mmtbx.command_line
cmdline = mmtbx.command_line.load_model_and_data(
args=args,
master_phil=get_master_phil(),
require_data=False,
create_fmodel=True,
process_pdb_file=True,
usage_string=usage_string,
prefer_anomalous=True,
out=out)
params = cmdline.params
fmodel = cmdline.fmodel
if (params.output.maps is Auto) and (fmodel is not None) :
params.output.maps = True
elif (params.output.maps == True) and (fmodel is None) :
raise Sorry("Map output requires experimental data.")
if (params.molprobity.keep_hydrogens is Auto) :
params.molprobity.keep_hydrogens = \
( (params.input.scattering_table == "neutron") or
(params.pdb_interpretation.use_neutron_distances) )
header_info = mmtbx.validation.molprobity.pdb_header_info(
pdb_file=params.input.pdb.file_name[0],
pdb_hierarchy=cmdline.pdb_hierarchy)
pdb_prefix = os.path.splitext(os.path.basename(
params.input.pdb.file_name[0]))[0]
if (params.output.prefix is None) :
params.output.prefix = "molprobity"
probe_file = None
if (params.output.probe_dots) or (params.output.kinemage) :
probe_file = params.output.prefix + "_probe.txt"
raw_data = cmdline.raw_data
# check map parameters
from mmtbx.real_space_correlation import check_map_file
check_map_file(None, params.input.maps)
validation = mmtbx.validation.molprobity.molprobity(
model=cmdline.model,
fmodel=fmodel,
flags=params.molprobity.flags,
sequences=cmdline.sequence,
raw_data=cmdline.raw_data,
unmerged_data=cmdline.unmerged_i_obs,
header_info=header_info,
keep_hydrogens=params.molprobity.keep_hydrogens,
nuclear=params.pdb_interpretation.use_neutron_distances,
save_probe_unformatted_file=probe_file,
min_cc_two_fofc=params.molprobity.min_cc_two_fofc,
n_bins_data=params.molprobity.n_bins,
outliers_only=params.molprobity.outliers_only,
use_pdb_header_resolution_cutoffs=\
params.molprobity.use_pdb_header_resolution_cutoffs,
count_anomalous_pairs_separately=\
params.molprobity.count_anomalous_pairs_separately,
use_internal_variance=params.input.unmerged_data.use_internal_variance,
file_name=params.input.pdb.file_name[0],
ligand_selection=params.molprobity.ligand_selection,
rotamer_library=params.molprobity.rotamer_library,
map_params=params)
map_file = None
# model cannot be pickled
validation.model = None
# polygon statistics
validation.polygon_stats = validation.get_polygon_statistics(
params.polygon.keys_to_show)
if ('pdb_header_r_work' in params.polygon.keys_to_show):
validation.polygon_stats['pdb_header_r_work'] = header_info.r_work
if ('pdb_header_r_free' in params.polygon.keys_to_show):
validation.polygon_stats['pdb_header_r_free'] = header_info.r_free
if (not params.output.quiet) :
out2 = multi_out()
out2.register("stdout", out)
f = open(params.output.prefix + ".out", "w")
out2.register("txt_out", f)
validation.show(out=out2,
outliers_only=params.molprobity.outliers_only,
show_percentiles=params.output.percentiles)
f.close()
print >> out, ""
print >> out, "Results written to %s.out" % params.output.prefix
if (params.output.kinemage) :
if (cmdline.pdb_hierarchy.models_size() == 1) :
assert (probe_file is not None)
import mmtbx.kinemage.validation
cmdline.pdb_hierarchy.atoms().reset_i_seq()
kin_file = "%s.kin" % params.output.prefix
kin_out = \
mmtbx.kinemage.validation.export_molprobity_result_as_kinemage(
result=validation,
pdb_hierarchy=cmdline.pdb_hierarchy,
geometry=cmdline.geometry,
probe_file=probe_file,
keep_hydrogens=params.molprobity.keep_hydrogens,
pdbID=pdb_prefix)
f = open(kin_file, "w")
f.write(kin_out)
f.close()
if (not params.output.quiet) :
print >> out, "Wrote kinemage to %s" % kin_file
else :
print >> out, "Kinemage output not available for multiple MODELs."
if (params.output.pickle) :
easy_pickle.dump("%s.pkl" % params.output.prefix, validation)
if (not params.output.quiet) :
print >> out, "Saved result to %s.pkl" % params.output.prefix
if (params.output.coot) :
coot_file = "%s_coot.py" % params.output.prefix
validation.write_coot_script(coot_file)
if (not params.output.quiet) :
print >> out, "Wrote script for Coot: %s" % coot_file
if (params.output.maps == True) :
import mmtbx.maps.utils
import iotbx.map_tools
map_file = "%s_maps.mtz" % params.output.prefix
two_fofc_map, fofc_map = mmtbx.maps.utils.get_maps_from_fmodel(
fmodel=fmodel,
fill_missing_f_obs=params.output.map_options.fill_missing_f_obs,
exclude_free_r_reflections=\
params.output.map_options.exclude_free_r_reflections)
anom_map = None
if (fmodel.f_obs().anomalous_flag()) :
anom_map = mmtbx.maps.utils.get_anomalous_map(fmodel)
iotbx.map_tools.write_map_coeffs(
file_name=map_file,
fwt_coeffs=two_fofc_map,
delfwt_coeffs=fofc_map,
anom_coeffs=anom_map)
print >> out, "Wrote map coefficients to %s" % map_file
else :
print >> out, ""
validation.show_summary(out=out, show_percentiles=params.output.percentiles)
if (params.output.wxplots) :
try :
import wxtbx.app
except ImportError, e :
raise Sorry("wxPython not available.")
else :
app = wxtbx.app.CCTBXApp(0)
validation.display_wx_plots()
app.MainLoop()
def find_merge_common_images(args):
phil = iotbx.phil.process_command_line(args = args,
master_string = master_phil).show()
work_params = phil.work.extract()
if ("--help" in args) :
libtbx.phil.parse(master_phil.show())
return
if ((work_params.d_min is None) or
(work_params.data is None) or
((work_params.model is None) and
work_params.scaling.algorithm != "mark1")) :
raise Usage("cxi.merge "
"d_min=4.0 "
"data=~/scratch/r0220/006/strong/ "
"model=3bz1_3bz2_core.pdb")
if ((work_params.rescale_with_average_cell) and
(not work_params.set_average_unit_cell)) :
raise Usage(
"If rescale_with_average_cell=True, you must also specify "+
"set_average_unit_cell=True.")
# Read Nat's reference model from an MTZ file. XXX The observation
# type is given as F, not I--should they be squared? Check with Nat!
log = open("%s_%s_scale.log" % (work_params.output.prefix,
work_params.scaling.algorithm), "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
print >> out, "Target unit cell and space group:"
print >> out, " ", work_params.target_unit_cell
print >> out, " ", work_params.target_space_group
uc = work_params.target_unit_cell
miller_set = symmetry(
unit_cell=work_params.target_unit_cell,
space_group_info=work_params.target_space_group
).build_miller_set(
anomalous_flag=not work_params.merge_anomalous,
d_min=work_params.d_min)
from xfel.cxi.merging.general_fcalc import random_structure
i_model = random_structure(work_params)
# ---- Augment this code with any special procedures for x scaling
scaler = xscaling_manager(
miller_set=miller_set,
i_model=i_model,
params=work_params,
log=out)
scaler.read_all()
print "finished reading"
sg = miller_set.space_group()
pg = sg.build_derived_laue_group()
miller_set.show_summary()
hkl_asu = scaler.observations["hkl_id"]
imageno = scaler.observations["frame_id"]
intensi = scaler.observations["i"]
sigma_i = scaler.observations["sigi"]
lookup = scaler.millers["merged_asu_hkl"]
# construct table of start / end indices for frames: now using Python
# range indexing
starts = [0]
ends = []
for x in xrange(1, len(scaler.observations["hkl_id"])):
if imageno[x] != imageno[x - 1]:
ends.append(x)
starts.append(x)
ends.append(len(scaler.observations["hkl_id"]))
keep_start = []
keep_end = []
for j, se in enumerate(zip(starts, ends)):
s, e = se
isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
dmin = 100.0
for x in xrange(s, e):
d = uc.d(lookup[hkl_asu[x]])
if d < dmin:
dmin = d
if isig > 6.0 and dmin < 3.2:
keep_start.append(s)
keep_end.append(e)
starts = keep_start
ends = keep_end
print 'Keeping %d frames' % len(starts)
frames = []
for s, e in zip(starts, ends):
indices = [tuple(lookup[hkl_asu[x]]) for x in range(s, e)]
intensities = intensi[s:e]
sigmas = sigma_i[s:e]
frames.append(Frame(uc, indices, intensities, sigmas))
cycle = 0
total_nref = sum([len(f.get_indices()) for f in frames])
# pre-scale the data - first determine average ln(k), B; then apply
kbs = [f.kb() for f in frames]
mn_k = sum([kb[0] for kb in kbs]) / len(kbs)
mn_B = sum([kb[1] for kb in kbs]) / len(kbs)
for f in frames:
f.scale_to_kb(mn_k, mn_B)
while True:
print 'Analysing %d frames' % len(frames)
print 'Cycle %d' % cycle
cycle += 1
print 'Power spectrum'
fn = frame_numbers(frames)
for j in sorted(fn):
print '%4d %4d' % (j, fn[j])
nref_cycle = sum([len(f.get_indices()) for f in frames])
assert(nref_cycle == total_nref)
common_reflections = numpy.zeros((len(frames), len(frames)),
dtype = numpy.short)
obs = { }
from cctbx.sgtbx import rt_mx, change_of_basis_op
oh = change_of_basis_op(rt_mx('h,l,k'))
for j, f in enumerate(frames):
indices = set(f.get_indices())
for i in indices:
_i = tuple(i)
if not _i in obs:
obs[_i] = []
obs[_i].append(j)
# work through unique observations ignoring those which include no
# hand information
for hkl in obs:
if hkl == oh.apply(hkl):
continue
obs[hkl].sort()
for j, f1 in enumerate(obs[hkl][:-1]):
for f2 in obs[hkl][j + 1:]:
common_reflections[(f1, f2)] += 1
cmn_rfl_list = []
for f1 in range(len(frames)):
for f2 in range(f1 + 1, len(frames)):
if common_reflections[(f1, f2)] > 20:
cmn_rfl_list.append((common_reflections[(f1, f2)], f1, f2))
cmn_rfl_list.sort()
cmn_rfl_list.reverse()
joins = []
used = []
for n, f1, f2 in cmn_rfl_list:
if f1 in used or f2 in used:
continue
_cc = frames[f1].cc(frames[f2])
# really only need to worry about f2 which will get merged...
# merging multiple files together should be OK provided they are
# correctly sorted (though the order should not matter anyhow?)
# anyhow they are sorted anyway... ah as f2 > f1 then just sorting
# the list by f2 will make sure the data cascase correctly.
# p-value very small for cc > 0.75 for > 20 observations - necessary
# as will be correlated due to Wilson curves
if _cc[0] > 20 and _cc[1] > 0.75:
print '%4d %.3f' % _cc, f1, f2
joins.append((f2, f1))
# used.append(f1)
used.append(f2)
if not joins:
print 'No pairs found'
break
joins.sort()
joins.reverse()
for j2, j1 in joins:
rmerge = frames[j1].merge(frames[j2])
if rmerge:
print 'R: %4d %4d %6.3f' % (j1, j2, rmerge)
else:
print 'R: %4d %4d ------' % (j1, j2)
continue
frames.sort()
print 'Biggest few: #frames; #unique refl'
j = -1
while frames[j].get_frames() > 1:
print frames[j].get_frames(), frames[j].get_unique_indices()
j -= 1
return
def set_log(prefix, i_current, i_total):
log = multi_out()
fo = open("%s.tls.%s_of_%s" % (prefix, str(i_current), i_total), "w")
log.register(label="log_buffer", file_object=fo)
sys.stderr = log
return log
def run(args):
if len(args)==0:
master_params.show(expert_level=0)
elif ( "--help" in args ):
print "no help available"
elif ( "--h" in args ):
print "no help available"
elif ( "--show_defaults" in args ):
master_params.show(expert_level=0)
elif ( "--show_defaults_all" in args ):
master_params.show(expert_level=10)
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
log_plots = StringIO()
print >> log,"#phil __OFF__"
print >> log
print >> log, date_and_time()
print >> log
print >> log
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="scaling")
reflection_file = None
for arg in args:
command_line_params = None
arg_is_processed = False
if arg == '--quiet':
arg_is_processed = True
## The associated action with this keyword is implemented above
if (os.path.isfile(arg)): ## is this a file name?
## Check if this is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
except KeyboardInterrupt: raise
except Exception : pass
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
## Check if this file is a reflection file
if command_line_params is None:
reflection_file = reflection_file_reader.any_reflection_file(
file_name=arg, ensure_read_access=False)
if (reflection_file is not None):
reflection_file = arg
arg_is_processed = True
## If it is not a file, it must be a phil command
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown phil-file or phil-command:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file format or phil command: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
## Now please read in the reflections files
## get symmetry and cell data first please
## By default, the native cell and symmetry are used
## as reference
crystal_symmetry_nat = None
crystal_symmetry_nat = crystal_symmetry_from_any.extract_from(
file_name=params.scaling.input.xray_data.after_burn.file_name)
if params.scaling.input.xray_data.space_group is None:
params.scaling.input.xray_data.space_group =\
crystal_symmetry_nat.space_group_info()
print >> log, "Using symmetry of after_burn data"
if params.scaling.input.xray_data.unit_cell is None:
params.scaling.input.xray_data.unit_cell =\
crystal_symmetry_nat.unit_cell()
print >> log, "Using cell of after_burn data"
## Check if a unit cell is defined
if params.scaling.input.xray_data.space_group is None:
raise Sorry("No space group defined")
if params.scaling.input.xray_data.unit_cell is None:
raise Sorry("No unit cell defined")
crystal_symmetry = crystal_symmetry = crystal.symmetry(
unit_cell = params.scaling.input.xray_data.unit_cell,
space_group_symbol = str(
params.scaling.input.xray_data.space_group) )
effective_params = master_params.fetch(sources=phil_objects)
new_params = master_params.format(python_object=params)
print >> log, "Effective parameters"
print >> log, "#phil __ON__"
new_params.show(out=log,
expert_level=params.scaling.input.expert_level )
print >> log, "#phil __END__"
print >> log
## define a xray data server
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = crystal_symmetry,
force_symmetry = True,
reflection_files=[])
## Read in native data and make appropriatre selections
miller_array_native = None
miller_array_native = xray_data_server.get_xray_data(
file_name = params.scaling.input.xray_data.after_burn.file_name,
labels = params.scaling.input.xray_data.after_burn.labels,
ignore_all_zeros = True,
parameter_scope = 'scaling.input.SIR_scale.xray_data.after_burn'
)
info_native = miller_array_native.info()
miller_array_native=miller_array_native.map_to_asu().select(
miller_array_native.indices()!=(0,0,0) )
miller_array_native = miller_array_native.select(
miller_array_native.data() > 0 )
## Convert to amplitudes
if (miller_array_native.is_xray_intensity_array()):
miller_array_native = miller_array_native.f_sq_as_f()
elif (miller_array_native.is_complex_array()):
miller_array_native = abs(miller_array_native)
if not miller_array_native.is_real_array():
raise Sorry("miller_array_native is not a real array")
miller_array_native.set_info(info = info_native)
## Read in derivative data and make appropriate selections
miller_array_derivative = None
miller_array_derivative = xray_data_server.get_xray_data(
file_name = params.scaling.input.xray_data.before_burn.file_name,
labels = params.scaling.input.xray_data.before_burn.labels,
ignore_all_zeros = True,
parameter_scope = 'scaling.input.SIR_scale.xray_data.before_burn'
)
info_derivative = miller_array_derivative.info()
miller_array_derivative=miller_array_derivative.map_to_asu().select(
miller_array_derivative.indices()!=(0,0,0) )
miller_array_derivative = miller_array_derivative.select(
miller_array_derivative.data() > 0 )
## Convert to amplitudes
if (miller_array_derivative.is_xray_intensity_array()):
miller_array_derivative = miller_array_derivative.f_sq_as_f()
elif (miller_array_derivative.is_complex_array()):
miller_array_derivative = abs(miller_array_derivative)
if not miller_array_derivative.is_real_array():
raise Sorry("miller_array_derivative is not a real array")
miller_array_derivative.set_info(info = info_derivative)
## As this is a SIR case, we will remove any anomalous pairs
if miller_array_derivative.anomalous_flag():
miller_array_derivative = miller_array_derivative.average_bijvoet_mates()\
.set_observation_type( miller_array_derivative )
if miller_array_native.anomalous_flag():
miller_array_native = miller_array_native.average_bijvoet_mates()\
.set_observation_type( miller_array_native )
## Print info
print >> log
print >> log, "Native data"
print >> log, "==========="
miller_array_native.show_comprehensive_summary(f=log)
print >> log
native_pre_scale = pre_scale.pre_scaler(
miller_array_native,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_native = native_pre_scale.x1.deep_copy()
del native_pre_scale
print >> log
print >> log, "Derivative data"
print >> log, "==============="
miller_array_derivative.show_comprehensive_summary(f=log)
print >> log
derivative_pre_scale = pre_scale.pre_scaler(
miller_array_derivative,
params.scaling.input.scaling_strategy.pre_scaler_protocol,
params.scaling.input.basic)
miller_array_derivative = derivative_pre_scale.x1.deep_copy()
del derivative_pre_scale
scaler = fa_estimation.combined_scaling(
miller_array_native,
miller_array_derivative,
params.scaling.input.scaling_strategy.iso_protocol)
miller_array_native = scaler.x1.deep_copy()
miller_array_derivative = scaler.x2.deep_copy()
del scaler
print >> log
print >> log, "Making delta f's"
print >> log, "----------------"
print >> log
delta_gen = pair_analyses.delta_generator( miller_array_native,
miller_array_derivative,
params.scaling.input.scaling_strategy.iso_protocol.nsr_bias )
print >> log
print >> log, "writing mtz file"
print >> log, "----------------"
print >> log
## some assertions to make sure nothing went weerd
assert miller_array_native.observation_type() is not None
assert miller_array_derivative.observation_type() is not None
assert delta_gen.abs_delta_f.observation_type() is not None
## Please write out the abs_delta_f array
mtz_dataset = delta_gen.abs_delta_f.as_mtz_dataset(
column_root_label='F'+params.scaling.input.output.outlabel)
mtz_dataset.mtz_object().write(
file_name=params.scaling.input.output.hklout)
def __init__(self,
cmd_list,
nprocs=1,
out=sys.stdout,
log=None,
verbosity=DEFAULT_VERBOSITY,
max_time=180):
if (log is None): log = null_out()
self.out = multi_out()
self.log = log
self.out.register("stdout", out)
self.out.register("log", log)
self.verbosity = verbosity
self.quiet = (verbosity == 0)
self.results = []
self.pool = None
# Filter cmd list for duplicates.
self.cmd_list = []
for cmd in cmd_list:
if (not cmd in self.cmd_list):
self.cmd_list.append(cmd)
else:
print >> self.out, "Test %s repeated, skipping" % cmd
# Set number of processors.
if (nprocs is Auto):
nprocs = cpu_count()
nprocs = min(nprocs, len(self.cmd_list))
# Starting summary.
if (self.verbosity > 0):
print >> self.out, "Running %d tests on %s processors:" % (len(
self.cmd_list), nprocs)
for cmd in self.cmd_list:
print >> self.out, " %s" % cmd
print >> self.out, ""
# Either run tests in parallel or run parallel tests, but
# can't run parallel tests in parallel (cctbx#95)
os.environ['OPENBLAS_NUM_THREADS'] = "1"
t_start = time.time()
if nprocs > 1:
# Run the tests with multiprocessing pool.
self.pool = Pool(processes=nprocs)
for command in self.cmd_list:
self.pool.apply_async(run_command, [command, verbosity, out],
callback=self.save_result)
try:
self.pool.close()
except KeyboardInterrupt:
print >> self.out, "Caught KeyboardInterrupt, terminating"
self.pool.terminate()
finally:
try:
self.pool.join()
except KeyboardInterrupt:
pass
else:
# Run tests serially.
for command in self.cmd_list:
rc = run_command(command, verbosity=verbosity, out=out)
if self.save_result(rc) == False:
break
# Print ending summary.
t_end = time.time()
print >> self.out, "=" * 80
print >> self.out, ""
print >> self.out, "Tests finished. Elapsed time: %.2fs" % (t_end -
t_start)
print >> self.out, ""
# Process results for errors and warnings.
extra_stderr = len(
[result for result in self.results if result.stderr_lines])
longjobs = [
result for result in self.results if result.wall_time > max_time
]
warnings = [
result for result in self.results
if result.alert_status == Status.WARNING
]
failures = [
result for result in self.results
if result.alert_status == Status.FAIL
]
self.finished = len(self.results)
self.failure = len(failures)
self.warning = len(warnings)
# Try writing the XML result file
write_JUnit_XML(self.results, "output.xml")
# Run time distribution.
if (libtbx.env.has_module("scitbx")):
from scitbx.array_family import flex
print >> self.out, "Distribution of test runtimes:"
hist = flex.histogram(flex.double(
[result.wall_time for result in self.results]),
n_slots=10)
hist.show(f=self.out, prefix=" ", format_cutoffs="%.1fs")
print >> self.out, ""
# Long job warning.
if longjobs:
print >> self.out, ""
print >> self.out, "Warning: the following jobs took at least %d seconds:" % max_time
for result in sorted(longjobs,
key=lambda result: result.wall_time):
print >> self.out, " %s: %.1fs" % (result.command,
result.wall_time)
else:
# Just print 5 worst offenders to encourage developers to check them out
print >> self.out, ""
print >> self.out, "Warning: the following are 5 longest jobs:"
for result in sorted(self.results,
key=lambda result: -result.wall_time)[:5]:
print >> self.out, " %s: %.1fs" % (result.command,
result.wall_time)
print >> self.out, "Please try to reduce overall runtime - consider splitting up these tests."
print >> self.out, ""
# Failures.
if failures:
print >> self.out, ""
print >> self.out, "Error: the following jobs returned non-zero exit codes or suspicious stderr output:"
print >> self.out, ""
for result in warnings:
self.display_result(result,
alert=Status.WARNING,
out=self.out,
log_return=self.out,
log_stderr=self.out)
for result in failures:
self.display_result(result,
alert=Status.FAIL,
out=self.out,
log_return=self.out,
log_stderr=self.out)
print >> self.out, ""
print >> self.out, "Please verify these tests manually."
print >> self.out, ""
# Summary
print >> self.out, "Summary:"
print >> self.out, " Tests run :", self.finished
print >> self.out, " Failures :", self.failure
print >> self.out, " Warnings (possible failures) :", self.warning
print >> self.out, " Stderr output (discouraged) :", extra_stderr
if (self.finished != len(self.cmd_list)):
print >> self.out, "*" * 80
print >> self.out, " WARNING: NOT ALL TESTS FINISHED!"
print >> self.out, "*" * 80
def run(args, command_name="phenix.remove_outliers"):
if (len(args)==0 or "--help" in args or "--h" in args or "-h" in args):
print_help(command_name=command_name)
else:
log = multi_out()
plot_out = None
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="outlier_detection")
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if arg=="--quiet":
arg_is_processed = True
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
if not os.path.exists( params.outlier_utils.input.xray_data.file_name ) :
raise Sorry("File %s can not be found"%(params.outlier_utils.input.xray_data.file_name) )
if params.outlier_utils.input.model.file_name is not None:
if not os.path.exists( params.outlier_utils.input.model.file_name ):
raise Sorry("File %s can not be found"%(params.outlier_utils.input.model.file_name) )
# now get the unit cell from the pdb file
hkl_xs = None
if params.outlier_utils.input.xray_data.file_name is not None:
hkl_xs = crystal_symmetry_from_any.extract_from(
file_name=params.outlier_utils.input.xray_data.file_name)
pdb_xs = None
if params.outlier_utils.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.outlier_utils.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.outlier_utils.input.unit_cell,
space_group_info=params.outlier_utils.input.space_group )
phil_xs.show_summary()
hkl_xs.show_summary()
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[hkl_xs])
# inject the unit cell and symmetry in the phil scope please
params.outlier_utils.input.unit_cell = combined_xs.unit_cell()
params.outlier_utils.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
new_params = master_params.format(python_object=params)
new_params.show(out=log)
if params.outlier_utils.input.unit_cell is None:
raise Sorry("unit cell not specified")
if params.outlier_utils.input.space_group is None:
raise Sorry("space group not specified")
if params.outlier_utils.input.xray_data.file_name is None:
raise Sorry("Xray data not specified")
if params.outlier_utils.input.model.file_name is None:
print "PDB file not specified. Basic wilson outlier rejections only."
#-----------------------------------------------------------
#
# step 1: read in the reflection file
#
phil_xs = crystal.symmetry(
unit_cell=params.outlier_utils.input.unit_cell,
space_group_info=params.outlier_utils.input.space_group )
xray_data_server = reflection_file_utils.reflection_file_server(
crystal_symmetry = phil_xs,
force_symmetry = True,
reflection_files=[])
miller_array = None
miller_array = xray_data_server.get_xray_data(
file_name = params.outlier_utils.input.xray_data.file_name,
labels = params.outlier_utils.input.xray_data.obs_labels,
ignore_all_zeros = True,
parameter_scope = 'outlier_utils.input.xray_data',
parameter_name = 'obs_labels'
)
info = miller_array.info()
miller_array = miller_array.map_to_asu()
miller_array = miller_array.select(
miller_array.indices() != (0,0,0))
#we have to check if the sigma's make any sense at all
if not miller_array.sigmas_are_sensible():
miller_array = miller_array.customized_copy(
data = miller_array.data(),
sigmas=None).set_observation_type(miller_array)
miller_array = miller_array.select(
miller_array.data() > 0 )
if miller_array.sigmas() is not None:
miller_array = miller_array.select(
miller_array.sigmas() > 0 )
if (miller_array.is_xray_intensity_array()):
miller_array = miller_array.f_sq_as_f()
elif (miller_array.is_complex_array()):
miller_array = abs(miller_array)
miller_array.set_info(info)
merged_anomalous=False
if miller_array.anomalous_flag():
miller_array = miller_array.average_bijvoet_mates().set_observation_type(
miller_array )
merged_anomalous=True
miller_array = miller_array.map_to_asu()
# get the free reflections please
free_flags = None
if params.outlier_utils.input.xray_data.free_flags is None:
free_flags = miller_array.generate_r_free_flags(
fraction=params.outlier_utils.\
additional_parameters.free_flag_generation.fraction,
max_free=params.outlier_utils.\
additional_parameters.free_flag_generation.max_number,
lattice_symmetry_max_delta=params.outlier_utils.\
additional_parameters.free_flag_generation.lattice_symmetry_max_delta,
use_lattice_symmetry=params.outlier_utils.\
additional_parameters.free_flag_generation.use_lattice_symmetry
)
else:
free_flags = xray_data_server.get_xray_data(
file_name = params.outlier_utils.input.xray_data.file_name,
labels = params.outlier_utils.input.xray_data.free_flags,
ignore_all_zeros = True,
parameter_scope = 'outlier_utils.input.xray_data',
parameter_name = 'free_flags'
)
if free_flags.anomalous_flag():
free_flags = free_flags.average_bijvoet_mates()
merged_anomalous=True
free_flags = free_flags.customized_copy(
data = flex.bool( free_flags.data() == 1 ))
free_flags = free_flags.map_to_asu()
free_flags = free_flags.common_set( miller_array )
print >> log
print >> log, "Summary info of observed data"
print >> log, "============================="
miller_array.show_summary(f=log)
if merged_anomalous:
print >> log, "For outlier detection purposes, the Bijvoet pairs have been merged."
print >> log
print >> log, "Constructing an outlier manager"
print >> log, "==============================="
print >> log
outlier_manager = outlier_rejection.outlier_manager(
miller_array,
free_flags,
out=log)
basic_array = None
extreme_array = None
model_based_array = None
basic_array = outlier_manager.basic_wilson_outliers(
p_basic_wilson = params.outlier_utils.outlier_detection.\
parameters.basic_wilson.level,
return_data = True)
extreme_array = outlier_manager.extreme_wilson_outliers(
p_extreme_wilson = params.outlier_utils.outlier_detection.parameters.\
extreme_wilson.level,
return_data = True)
beamstop_array = outlier_manager.beamstop_shadow_outliers(
level = params.outlier_utils.outlier_detection.parameters.\
beamstop.level,
d_min = params.outlier_utils.outlier_detection.parameters.\
beamstop.d_min,
return_data=True)
#----------------------------------------------------------------
# Step 2: get an xray structure from the PDB file
#
if params.outlier_utils.input.model.file_name is not None:
model = pdb.input(file_name=params.outlier_utils.input.model.file_name).xray_structure_simple(
crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary(f=log)
print >> log
# please make an f_model object for bulk solvent scaling etc etc
f_model_object = f_model.manager(
f_obs = miller_array,
r_free_flags = free_flags,
xray_structure = model )
print >> log, "Bulk solvent scaling of the data"
print >> log, "================================"
print >> log, "Maximum likelihood bulk solvent scaling."
print >> log
f_model_object.update_solvent_and_scale(out=log)
plot_out = StringIO()
model_based_array = outlier_manager.model_based_outliers(
f_model_object.f_model(),
level=params.outlier_utils.outlier_detection.parameters.model_based.level,
return_data=True,
plot_out=plot_out)
#check what needs to be put out please
if params.outlier_utils.output.hklout is not None:
if params.outlier_utils.outlier_detection.protocol == "model":
if params.outlier_utils.input.model.file_name == None:
print >> log, "Model based rejections requested. No model was supplied."
print >> log, "Switching to writing out rejections based on extreme value Wilson statistics."
params.outlier_utils.outlier_detection.protocol="extreme"
output_array = None
print >> log
if params.outlier_utils.outlier_detection.protocol == "basic":
print >> log, "Non-outliers found by the basic wilson statistics"
print >> log, "protocol will be written out."
output_array = basic_array
new_set_of_free_flags = free_flags.common_set( basic_array )
if params.outlier_utils.outlier_detection.protocol == "extreme":
print >> log, "Non-outliers found by the extreme value wilson statistics"
print >> log, "protocol will be written out."
output_array = extreme_array
new_set_of_free_flags = free_flags.common_set( extreme_array )
if params.outlier_utils.outlier_detection.protocol == "model":
print >> log, "Non-outliers found by the model based"
print >> log, "protocol will be written out to the file:"
print >> log, params.outlier_utils.output.hklout
print >> log
output_array = model_based_array
new_set_of_free_flags = free_flags.common_set( model_based_array )
if params.outlier_utils.outlier_detection.protocol == "beamstop":
print >> log, "Outliers found for the beamstop shadow"
print >> log, "problems detection protocol will be written to the file:"
print >> log, params.outlier_utils.output.hklout
print >> log
output_array = model_based_array
new_set_of_free_flags = free_flags.common_set( model_based_array )
mtz_dataset = output_array.as_mtz_dataset(
column_root_label="FOBS")
mtz_dataset = mtz_dataset.add_miller_array(
miller_array = new_set_of_free_flags,
column_root_label = "Free_R_Flag"
)
mtz_dataset.mtz_object().write(
file_name=params.outlier_utils.output.hklout)
if (params.outlier_utils.output.logfile is not None):
final_log = StringIO()
print >> final_log, string_buffer.getvalue()
print >> final_log
if plot_out is not None:
print >> final_log, plot_out.getvalue()
outfile = open( params.outlier_utils.output.logfile, 'w' )
outfile.write( final_log.getvalue() )
print >> log
print >> log, "A logfile named %s was created."%(
params.outlier_utils.output.logfile)
print >> log, "This logfile contains the screen output and"
print >> log, "(possibly) some ccp4 style loggraph plots"
def __init__(self,
datatypes=None,
phil=None,
custom_options=None,
logger=None):
'''
Base DataManager class
'''
self.datatypes = datatypes
if self.datatypes is None:
self.datatypes = []
if hasattr(self, 'datatype') and (self.datatype not in self.datatypes):
self.datatypes.append(self.datatype)
# custom data processing options
self.custom_options = custom_options
if self.custom_options is not None:
for option in self.custom_options:
if option not in data_manager_options:
raise Sorry('''\
{option} is not a valid custom option for the DataManager. The available
options are {options}.\
'''.format(option=option, options=', '.join(data_manager_options)))
else:
self.custom_options = []
# functions for custom PHIL
self.add_custom_phil_str = 'add_%s_phil_str'
self.export_custom_phil_extract = 'export_%s_phil_extract'
self.load_custom_phil_extract = 'load_%s_phil_extract'
# dynamically construct master PHIL string
self.master_phil_str = 'data_manager {\n'
for datatype in self.datatypes:
# check if a datatype has a custom PHIL str
if hasattr(self, self.add_custom_phil_str % datatype):
custom_phil_str = getattr(self, self.add_custom_phil_str %
datatype)()
self.master_phil_str += custom_phil_str
# default PHIL
else:
# sequence_files = None
# .type = path
# .multiple = True
self.master_phil_str += '%s_files = None\n' % datatype
self.master_phil_str += '.type = path\n.multiple=True\n'
# property for wx GUI (will be removed)
file_type = any_file_type.get(datatype, None)
if file_type is not None:
self.master_phil_str += '.style = file_type:%s input_file\n' % \
file_type
# default_sequence = None
# .type = path
self.master_phil_str += 'default_%s = None\n' % datatype
self.master_phil_str += '.type = path\n'
self.master_phil_str += '}'
self.master_phil = libtbx.phil.parse(self.master_phil_str)
self._storage = '_%ss'
self._default = '_default_%s'
self._current_storage = None
self._current_default = None
# generate storage for each data type
for datatype in self.datatypes:
# data attributes for each data type
# e.g self._models, self._default_model
self._set_datatype(datatype)
setattr(self, self._current_storage, OrderedDict())
setattr(self, self._current_default, None)
# track output files for internal use
self._output_files = []
self._output_types = []
# set defaults
self._default_output_filename = 'cctbx_program'
self._overwrite = False
self._used_output_ext = set()
# set program
self._program = None
# logger (currently used for models)
self.logger = logger
if self.logger is None:
self.logger = multi_out()
# load information from phil
if phil is not None:
self.load_phil_scope(phil)
def sfcalc(args):
if len(args)==0:
print_help()
elif ( "--help" in args ):
print_help()
elif ( "--h" in args ):
print_help()
elif ("-h" in args ):
print_help()
else:
log = multi_out()
if (not "--quiet" in args):
log.register(label="stdout", file_object=sys.stdout)
string_buffer = StringIO()
string_buffer_plots = StringIO()
log.register(label="log_buffer", file_object=string_buffer)
phil_objects = []
argument_interpreter = master_params.command_line_argument_interpreter(
home_scope="sfcalc")
print >> log, "#phil __OFF__"
print >> log, "================="
print >> log, " SFCALC "
print >> log, "================="
print >> log
for arg in args:
command_line_params = None
arg_is_processed = False
# is it a file?
if (os.path.isfile(arg)): ## is this a file name?
# check if it is a phil file
try:
command_line_params = iotbx.phil.parse(file_name=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
else:
try:
command_line_params = argument_interpreter.process(arg=arg)
if command_line_params is not None:
phil_objects.append(command_line_params)
arg_is_processed = True
except KeyboardInterrupt: raise
except Exception : pass
if not arg_is_processed:
print >> log, "##----------------------------------------------##"
print >> log, "## Unknown file or keyword:", arg
print >> log, "##----------------------------------------------##"
print >> log
raise Sorry("Unknown file or keyword: %s" % arg)
effective_params = master_params.fetch(sources=phil_objects)
params = effective_params.extract()
# now get the unit cell from the files
hkl_xs = None
pdb_xs = None
if params.sfcalc.input.model.file_name is not None:
pdb_xs = crystal_symmetry_from_any.extract_from(
file_name=params.sfcalc.input.model.file_name)
phil_xs = crystal.symmetry(
unit_cell=params.sfcalc.input.unit_cell,
space_group_info=params.sfcalc.input.space_group )
combined_xs = crystal.select_crystal_symmetry(
None,phil_xs, [pdb_xs],[None])
combined_xs.show_summary()
if combined_xs.unit_cell() is None:
raise Sorry("Unit cell not defined")
if combined_xs.space_group() is None:
raise Sorry("Space group not defined")
# inject the unit cell and symmetry in the phil scope please
params.sfcalc.input.unit_cell = combined_xs.unit_cell()
params.sfcalc.input.space_group = \
sgtbx.space_group_info( group = combined_xs.space_group() )
print >> log, "#phil __ON__"
new_params = master_params.format(python_object=params)
new_params.show(out=log)
print >> log, "#phil __END__"
pdb_model = None
if params.sfcalc.input.model.file_name is not None:
pdb_model = pdb.input(file_name=params.sfcalc.input.model.file_name)
model = pdb_model.xray_structure_simple(crystal_symmetry=phil_xs)
print >> log, "Atomic model summary"
print >> log, "===================="
model.show_summary()
print >> log
#make an f_model object please
b_cart = params.sfcalc.parameters.overall.b_cart
b_cart = [b_cart.b_11,
b_cart.b_22,
b_cart.b_33,
b_cart.b_12,
b_cart.b_13,
b_cart.b_23]
dummy = abs(model.structure_factors(
d_min = params.sfcalc.parameters.d_min,
anomalous_flag = False).f_calc())
flags = dummy.generate_r_free_flags(fraction = 0.1,
max_free = 99999999)
fmodel = f_model.manager( xray_structure = model,
r_free_flags = flags,
target_name = "ls_wunit_k1",
f_obs = dummy,
b_cart = b_cart,
k_sol = params.sfcalc.parameters.solvent.k_sol,
b_sol = params.sfcalc.parameters.solvent.b_sol )
calc_data_with_solvent_contrib = fmodel.f_model()
calc_data_with_solvent_contrib = calc_data_with_solvent_contrib.array(
data=calc_data_with_solvent_contrib.data()*params.sfcalc.parameters.overall.k_overall)
result = None
label = None
if params.sfcalc.parameters.output_type == "complex":
result = calc_data_with_solvent_contrib
label="FMODEL"
if params.sfcalc.parameters.output_type == "amplitudes":
result = abs(calc_data_with_solvent_contrib)
label="FMODEL"
if params.sfcalc.parameters.output_type == "intensities":
result = abs(calc_data_with_solvent_contrib)
result = result.f_as_f_sq()
label="IMODEL"
#write an mtz file with the data
mtz_dataset = result.as_mtz_dataset(
column_root_label=label)
mtz_dataset.mtz_object().write(
file_name=params.sfcalc.output.hklout)
#write the logfile
logger = open( params.sfcalc.output.logfile, 'w')
print >> log, "writing log file with name %s"%(params.sfcalc.output.logfile)
print >> log
print >> logger, string_buffer.getvalue()