def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) +
" [options] directory|file...").enable_chunk(
easy_all=True).enable_multiprocessing()).process(args=args,
min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print "TIME BEGIN cod_refine:", date_and_time()
print
#
master_phil = get_master_phil()
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print
params = work_phil.extract()
#
qi_dict = {}
all_pickles = []
for arg in remaining_args:
if (op.isdir(arg)):
for node in sorted(os.listdir(arg)):
if (node.endswith(".pickle")):
all_pickles.append(op.join(arg, node))
elif (node.startswith("qi_") and len(node) == 10):
qi = open(op.join(arg, node)).read().splitlines()
if (len(qi) == 1):
cod_id = node[3:]
quick_info = eval(qi[0])
assert cod_id not in qi_dict
qi_dict[cod_id] = quick_info
elif (op.isfile(arg)):
all_pickles.append(arg)
else:
raise RuntimeError("Not a file or directory: %s" % arg)
print "Number of pickle files:", len(all_pickles)
print "Number of quick_infos:", len(qi_dict)
sort_choice = params.sorting_of_pickle_files
if (len(qi_dict) != 0 and sort_choice is not None):
print "Sorting pickle files by n_atoms * n_refl:", sort_choice
assert sort_choice in ["down", "up"]
def sort_pickle_files():
if (sort_choice == "down"): i_sign = -1
else: i_sign = 1
buffer = []
for i, path in enumerate(all_pickles):
cod_id = op.basename(path).split(".", 1)[0]
qi = qi_dict.get(cod_id)
if (qi is None): nn = 2**31
else: nn = qi[0] * qi[1] * qi[2]
buffer.append((nn, i_sign * i, path))
buffer.sort()
if (i_sign < 0):
buffer.reverse()
result = []
for elem in buffer:
result.append(elem[-1])
return result
all_pickles = sort_pickle_files()
print
#
rss = params.random_subset.size
if (rss is not None and rss > 0):
seed = params.random_subset.seed
print "Selecting subset of %d pickle files using random seed %d" % (
rss, seed)
mt = flex.mersenne_twister(seed=seed)
perm = mt.random_permutation(size=len(all_pickles))[:rss]
flags = flex.bool(len(all_pickles), False).set_selected(perm, True)
all_pickles = flex.select(all_pickles, permutation=flags.iselection())
print
#
from libtbx.path import makedirs_race
if (params.wdir_root is not None):
makedirs_race(path=params.wdir_root)
if (params.pickle_refined_dir is not None):
makedirs_race(path=params.pickle_refined_dir)
#
n_caught = 0
for i_pickle, pickle_file_name in enumerate(all_pickles):
if (i_pickle % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, pickle_file_name)
except KeyboardInterrupt:
print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt"
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s" % pickle_file_name
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
print "done_with: %s (%.2f seconds)" % (pickle_file_name,
tm.elapsed())
print
sys.stdout.flush()
print
print "Number of exceptions caught:", n_caught
#
show_times()
print
print "TIME END cod_refine:", date_and_time()
def run(args, command_name):
from iotbx.option_parser import option_parser as iotbx_option_parser
from libtbx import easy_pickle
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_line = (
iotbx_option_parser(usage=command_name + " [options] [cod_id...]")
.enable_chunk(easy_all=True)
.enable_multiprocessing()
.option(None, "--max_atoms", type="int", default=99, metavar="INT")
.option(None, "--min_distance", type="float", default=0.5, metavar="FLOAT")
.option(None, "--i_obs_i_calc_plot", action="store_true", default=False)
.option(None, "--f_obs_f_calc_plot", action="store_true", default=False)
.option(None, "--max_fan_outlier_fraction", type="float", default=0.05, metavar="FLOAT")
.option(None, "--fan_offset_low", type="float", default=0.05, metavar="FLOAT")
.option(None, "--fan_offset_high", type="float", default=0.10, metavar="FLOAT")
.option(None, "--min_f_obs_f_calc_correlation", type="float", default=0.99, metavar="FLOAT")
.option(None, "--at_least_one_special_position", action="store_true", default=False)
.option(None, "--pickle_dir", type="str", default="cod_ma_xs", metavar="PATH")
).process(args=args)
if command_line.run_multiprocessing_chunks_if_applicable(command_call=[command_name]):
show_times()
return
co = command_line.options
#
from iotbx.cif import cod_tools
hkl_cif = cod_tools.build_hkl_cif(cod_ids=command_line.args)
hkl_cif.show_summary()
#
pickle_dir = co.pickle_dir
if co.at_least_one_special_position:
pickle_dir += "_special"
if not op.isdir(pickle_dir):
from libtbx.path import makedirs_race
makedirs_race(path=pickle_dir)
n_caught = 0
for i_pair, pair in enumerate(hkl_cif.hkl_cif_pairs.values()):
cod_id = op.basename(pair[0])[:-4]
if i_pair % command_line.chunk.n != command_line.chunk.i:
continue
try:
cd = cod_data(cod_id=cod_id, hkl_cif_pair=pair)
except KeyboardInterrupt:
print "CAUGHT EXCEPTION: KeyboardInterrupt"
return
except Exception, e:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s: %s: %s" % (command_name, cod_id, str(e))
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
if cd.is_useful(co):
easy_pickle.dump(
file_name="%s/%s.pickle" % (pickle_dir, cod_id), obj=(cd.c_obs, cd.xray_structure, cd.edge_list)
)
print >> open("%s/qi_%s" % (pickle_dir, cod_id), "w"), cd.quick_info()
else:
print "filtering out:", cod_id
print "done_with:", cod_id
print
def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) + " [options] directory|file...")
.enable_chunk(easy_all=True)
.enable_multiprocessing()
).process(args=args, min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print "TIME BEGIN cod_refine:", date_and_time()
print
#
master_phil = get_master_phil()
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print
params = work_phil.extract()
#
qi_dict = {}
all_pickles = []
for arg in remaining_args:
if (op.isdir(arg)):
for node in sorted(os.listdir(arg)):
if (node.endswith(".pickle")):
all_pickles.append(op.join(arg, node))
elif (node.startswith("qi_") and len(node) == 10):
qi = open(op.join(arg, node)).read().splitlines()
if (len(qi) == 1):
cod_id = node[3:]
quick_info = eval(qi[0])
assert cod_id not in qi_dict
qi_dict[cod_id] = quick_info
elif (op.isfile(arg)):
all_pickles.append(arg)
else:
raise RuntimeError("Not a file or directory: %s" % arg)
print "Number of pickle files:", len(all_pickles)
print "Number of quick_infos:", len(qi_dict)
sort_choice = params.sorting_of_pickle_files
if (len(qi_dict) != 0 and sort_choice is not None):
print "Sorting pickle files by n_atoms * n_refl:", sort_choice
assert sort_choice in ["down", "up"]
def sort_pickle_files():
if (sort_choice == "down"): i_sign = -1
else: i_sign = 1
buffer = []
for i,path in enumerate(all_pickles):
cod_id = op.basename(path).split(".",1)[0]
qi = qi_dict.get(cod_id)
if (qi is None): nn = 2**31
else: nn = qi[0] * qi[1] * qi[2]
buffer.append((nn, i_sign*i, path))
buffer.sort()
if (i_sign < 0):
buffer.reverse()
result = []
for elem in buffer:
result.append(elem[-1])
return result
all_pickles = sort_pickle_files()
print
#
rss = params.random_subset.size
if (rss is not None and rss > 0):
seed = params.random_subset.seed
print "Selecting subset of %d pickle files using random seed %d" % (
rss, seed)
mt = flex.mersenne_twister(seed=seed)
perm = mt.random_permutation(size=len(all_pickles))[:rss]
flags = flex.bool(len(all_pickles), False).set_selected(perm, True)
all_pickles = flex.select(all_pickles, permutation=flags.iselection())
print
#
from libtbx.path import makedirs_race
if (params.wdir_root is not None):
makedirs_race(path=params.wdir_root)
if (params.pickle_refined_dir is not None):
makedirs_race(path=params.pickle_refined_dir)
#
n_caught = 0
for i_pickle,pickle_file_name in enumerate(all_pickles):
if (i_pickle % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, pickle_file_name)
except KeyboardInterrupt:
print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt"
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s" % pickle_file_name
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
print "done_with: %s (%.2f seconds)" % (pickle_file_name, tm.elapsed())
print
sys.stdout.flush()
print
print "Number of exceptions caught:", n_caught
#
show_times()
print
print "TIME END cod_refine:", date_and_time()
def run(args, command_name):
from iotbx.option_parser import option_parser as iotbx_option_parser
from libtbx import easy_pickle
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_line = (iotbx_option_parser(
usage=command_name+" [options] [cod_id...]")
.enable_chunk(easy_all=True)
.enable_multiprocessing()
.option(None, "--max_atoms",
type="int",
default=99,
metavar="INT")
.option(None, "--min_distance",
type="float",
default=0.5,
metavar="FLOAT")
.option(None, "--i_obs_i_calc_plot",
action="store_true",
default=False)
.option(None, "--f_obs_f_calc_plot",
action="store_true",
default=False)
.option(None, "--max_fan_outlier_fraction",
type="float",
default=0.05,
metavar="FLOAT")
.option(None, "--fan_offset_low",
type="float",
default=0.05,
metavar="FLOAT")
.option(None, "--fan_offset_high",
type="float",
default=0.10,
metavar="FLOAT")
.option(None, "--min_f_obs_f_calc_correlation",
type="float",
default=0.99,
metavar="FLOAT")
.option(None, "--at_least_one_special_position",
action="store_true",
default=False)
.option(None, "--pickle_dir",
type="str",
default="cod_ma_xs",
metavar="PATH")
).process(args=args)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=[command_name])):
show_times()
return
co = command_line.options
#
from iotbx.cif import cod_tools
hkl_cif = cod_tools.build_hkl_cif(cod_ids=command_line.args)
hkl_cif.show_summary()
#
pickle_dir = co.pickle_dir
if (co.at_least_one_special_position):
pickle_dir += "_special"
if (not op.isdir(pickle_dir)):
from libtbx.path import makedirs_race
makedirs_race(path=pickle_dir)
n_caught = 0
for i_pair,pair in enumerate(hkl_cif.hkl_cif_pairs.values()):
cod_id = op.basename(pair[0])[:-4]
if (i_pair % command_line.chunk.n != command_line.chunk.i): continue
try:
cd = cod_data(cod_id=cod_id, hkl_cif_pair=pair)
except KeyboardInterrupt:
print("CAUGHT EXCEPTION: KeyboardInterrupt")
return
except Exception as e:
sys.stdout.flush()
print("CAUGHT EXCEPTION: %s: %s: %s" % (command_name, cod_id, str(e)), file=sys.stderr)
traceback.print_exc()
print(file=sys.stderr)
sys.stderr.flush()
n_caught += 1
else:
if (cd.is_useful(co)):
easy_pickle.dump(
file_name="%s/%s.pickle" % (pickle_dir, cod_id),
obj=(cd.c_obs, cd.xray_structure, cd.edge_list))
print(cd.quick_info(), file=open("%s/qi_%s" % (pickle_dir, cod_id), "w"))
else:
print("filtering out:", cod_id)
print("done_with:", cod_id)
print()
print()
print("Number of exceptions caught:", n_caught)
#
show_times()
