def __init__(self, data, lattice_id, resort=False, verbose=True):
self.verbose = verbose
###### INPUTS #######
# data = miller array: ASU miller index & intensity
# lattice_id = flex double: assignment of each miller index to a lattice number
######################
if resort:
order = flex.sort_permutation(lattice_id.data())
sorted_lattice_id = flex.select(lattice_id.data(), order)
sorted_data = data.data().select( order)
sorted_indices = data.indices().select( order)
self.lattice_id = sorted_lattice_id
self.data = data.customized_copy(indices = sorted_indices, data = sorted_data)
else:
self.lattice_id = lattice_id.data() # type flex int
self.data = data # type miller array with flex double data
assert type(self.data.indices()) == type(flex.miller_index())
assert type(self.data.data()) == type(flex.double())
# construct a lookup for the separate lattices
last_id = -1; self.lattices = flex.int()
for n in xrange(len(self.lattice_id)):
if self.lattice_id[n] != last_id:
last_id = self.lattice_id[n]
self.lattices.append(n)
def __init__(self, data, lattice_id, resort=False, verbose=True):
self.verbose = verbose
###### INPUTS #######
# data = miller array: ASU miller index & intensity
# lattice_id = flex double: assignment of each miller index to a lattice number
######################
if resort:
order = flex.sort_permutation(lattice_id.data())
sorted_lattice_id = flex.select(lattice_id.data(), order)
sorted_data = data.data().select(order)
sorted_indices = data.indices().select(order)
self.lattice_id = sorted_lattice_id
self.data = data.customized_copy(indices=sorted_indices,
data=sorted_data)
else:
self.lattice_id = lattice_id.data() # type flex int
self.data = data # type miller array with flex double data
assert type(self.data.indices()) == type(flex.miller_index())
assert type(self.data.data()) == type(flex.double())
# construct a lookup for the separate lattices
last_id = -1
self.lattices = flex.int()
for n in xrange(len(self.lattice_id)):
if self.lattice_id[n] != last_id:
last_id = self.lattice_id[n]
self.lattices.append(n)
class fixed_u_eq_adp_test_case(adp_restraints_test_case):
proxies = fixed_u_eq_adp_restraints(
xray_structure=smtbx.development.sucrose(), u_eq_ideal=0.025).proxies
# no need to test all of them every time
proxies = adp.shared_fixed_u_eq_adp_proxy(
flex.select(proxies, flags=flex.random_bool(proxies.size(), 0.5)))
manager = restraints.manager(fixed_u_eq_adp_proxies=proxies)
def restraint(self, proxy, u_iso=None, u_cart=None):
if u_cart is None:
u_cart = self.xray_structure.scatterers().extract_u_cart(
self.xray_structure.unit_cell())
return adp.fixed_u_eq_adp(adp_restraint_params(u_cart=u_cart), proxy)
def sorted_type_index_pairs(self, heaviest_first=True):
ugs = self.unique_gaussians_as_list()
pairs = []
sf0s = flex.double()
for t,i in self.type_index_pairs_as_dict().items():
pairs.append((t,i))
gaussian = ugs[i]
if (gaussian is None):
sf0s.append(0)
else:
sf0s.append(gaussian.at_stol(0))
perm = flex.sort_permutation(sf0s, reverse=heaviest_first)
return flex.select(pairs, permutation=perm)
class rigid_bond_test_case(adp_restraints_test_case):
proxies = rigid_bond_restraints(
xray_structure=smtbx.development.sucrose()).proxies
# no need to test all of them every time
proxies = adp.shared_rigid_bond_proxy(
flex.select(proxies, flags=flex.random_bool(proxies.size(), 0.3)))
manager = restraints.manager(rigid_bond_proxies=proxies)
def restraint(self, proxy, u_iso=None, u_cart=None):
if u_cart is None:
u_cart = self.xray_structure.scatterers().extract_u_cart(
self.xray_structure.unit_cell())
sites_cart = self.xray_structure.sites_cart()
return adp.rigid_bond(
adp_restraint_params(sites_cart=sites_cart, u_cart=u_cart), proxy)
class adp_similarity_test_case(adp_restraints_test_case):
proxies = adp_similarity_restraints(
xray_structure=smtbx.development.sucrose()).proxies
# no need to test all of them every time
proxies = adp.shared_adp_similarity_proxy(
flex.select(proxies, flags=flex.random_bool(proxies.size(), 0.5)))
manager = restraints.manager(adp_similarity_proxies=proxies)
def restraint(self, proxy, u_iso=None, u_cart=None):
if u_cart is None:
u_cart=self.xray_structure.scatterers().extract_u_cart(
self.xray_structure.unit_cell())
if u_iso is None:
u_iso=self.xray_structure.scatterers().extract_u_iso()
use_u_aniso=self.xray_structure.use_u_aniso()
return adp.adp_similarity(
adp_restraint_params(u_cart=u_cart, u_iso=u_iso, use_u_aniso=use_u_aniso),
proxy)
def cross_check(args):
quick_summaries = []
for file_name in args:
quick_summaries.append(easy_pickle.load(file_name))
assert len(quick_summaries) == 2
lines = []
max_of_errors = flex.double()
atomic_numbers = flex.double()
n_less = 0
n_greater = 0
n_equal = 0
for label_1,error_1 in quick_summaries[0].items():
error_2 = quick_summaries[1].get(label_1, None)
if (error_2 is not None):
line = "%-10s %7.4f %7.4f" % (label_1, error_1, error_2)
if (error_1 < error_2):
line += " less %7.4f" % (error_2/error_1)
n_less += 1
elif (error_1 > error_2):
line += " greater %7.4f" % (error_1/error_2)
n_greater += 1
else:
line += " equal"
n_equal += 1
lines.append(line)
max_of_errors.append(max(error_1, error_2))
atomic_numbers.append(
tiny_pse.table(label_1.split("_")[0]).atomic_number())
for sort_key,reverse in [(max_of_errors,True), (atomic_numbers,False)]:
perm = flex.sort_permutation(data=sort_key, reverse=reverse)
perm_lines = flex.select(lines, perm)
for line in perm_lines:
print line
print
print "n_less:", n_less
print "n_greater:", n_greater
print "n_equal:", n_equal
print "total:", n_less + n_greater + n_equal
def cross_check(args):
quick_summaries = []
for file_name in args:
quick_summaries.append(easy_pickle.load(file_name))
assert len(quick_summaries) == 2
lines = []
max_of_errors = flex.double()
atomic_numbers = flex.double()
n_less = 0
n_greater = 0
n_equal = 0
for label_1, error_1 in quick_summaries[0].items():
error_2 = quick_summaries[1].get(label_1, None)
if (error_2 is not None):
line = "%-10s %7.4f %7.4f" % (label_1, error_1, error_2)
if (error_1 < error_2):
line += " less %7.4f" % (error_2 / error_1)
n_less += 1
elif (error_1 > error_2):
line += " greater %7.4f" % (error_1 / error_2)
n_greater += 1
else:
line += " equal"
n_equal += 1
lines.append(line)
max_of_errors.append(max(error_1, error_2))
atomic_numbers.append(
tiny_pse.table(label_1.split("_")[0]).atomic_number())
for sort_key, reverse in [(max_of_errors, True), (atomic_numbers, False)]:
perm = flex.sort_permutation(data=sort_key, reverse=reverse)
perm_lines = flex.select(lines, perm)
for line in perm_lines:
print line
print
print "n_less:", n_less
print "n_greater:", n_greater
print "n_equal:", n_equal
print "total:", n_less + n_greater + n_equal
def random_elements(size, choices=["O", "Mg", "Si", "Ca"]):
return flex.select(
sequence=choices,
permutation=flex.random_size_t(size=size) % len(choices))
def run(gaussian_fit_pickle_file_names, itvc_file_name, kissel_dir):
itvc_tab = None
if (itvc_file_name is not None):
itvc_tab = itvc_section61_io.read_table6111(itvc_file_name)
fits = read_pickled_fits(gaussian_fit_pickle_file_names)
#easy_pickle.dump("all_fits.pickle", fits)
for k, v in fits.parameters.items():
print "# %s:" % k, v
print
max_errors = flex.double()
labeled_fits = []
n_processed = 0
for label in expected_labels(kissel_dir):
try:
fit_group = fits.all[label]
except Exception:
print "# Warning: Missing scattering_type:", label
else:
print "scattering_type:", label
prev_fit = None
for fit in fit_group:
if (prev_fit is not None):
if (fit.stol > prev_fit.stol):
print "# Warning: decreasing stol"
elif (fit.stol == prev_fit.stol):
if (fit.max_error < prev_fit.max_error):
print "# Warning: same stol but previous has larger error"
prev_fit = fit
fit.sort().show()
gaussian_fit = None
if (itvc_tab is not None and label != "O2-"):
entry = itvc_tab.entries[label]
sel = international_tables_stols <= fit.stol + 1.e-6
gaussian_fit = scitbx.math.gaussian.fit(
international_tables_stols.select(sel),
entry.table_y.select(sel),
entry.table_sigmas.select(sel), fit)
elif (kissel_dir is not None):
file_name = os.path.join(
kissel_dir, "%02d_%s_rf" %
(tiny_pse.table(label).atomic_number(), label))
tab = kissel_io.read_table(file_name)
sel = tab.itvc_sampling_selection() & (tab.x <=
fit.stol + 1.e-6)
gaussian_fit = scitbx.math.gaussian.fit(
tab.x.select(sel), tab.y.select(sel),
tab.sigmas.select(sel), fit)
if (gaussian_fit is not None):
max_errors.append(
flex.max(gaussian_fit.significant_relative_errors()))
labeled_fits.append(labeled_fit(label, gaussian_fit))
n_processed += 1
print
if (n_processed != len(fits.all)):
print "# Warning: %d fits were not processed." % (len(fits.all) -
n_processed)
print
if (max_errors.size() > 0):
print "Summary:"
perm = flex.sort_permutation(data=max_errors, reverse=True)
max_errors = max_errors.select(perm)
labeled_fits = flex.select(labeled_fits, perm)
quick_summary = {}
for me, lf in zip(max_errors, labeled_fits):
print lf.label, "n_terms=%d max_error: %.4f" % (
lf.gaussian_fit.n_terms(), me)
quick_summary[lf.label + "_" + str(lf.gaussian_fit.n_terms())] = me
if (me > 0.01):
fit = lf.gaussian_fit
re = fit.significant_relative_errors()
for s, y, a, r in zip(fit.table_x(), fit.table_y(),
fit.fitted_values(), re):
comment = ""
if (r > 0.01): comment = " large error"
print "%4.2f %7.4f %7.4f %7.4f %7.4f%s" % (s, y, a, a - y,
r, comment)
print
print
print " %2d:" % (i_0 + 1), cache_0.input.info()
print "No comparison."
print
else:
ccs = flex.double()
for cb_op in unique_reindexing_operators:
similar_array_0 = cache_0.observations \
.change_basis(cb_op) \
.map_to_asu()
ccs.append(
cache_1.observations.correlation(
other=similar_array_0,
assert_is_similar_symmetry=False).coefficient())
permutation = flex.sort_permutation(ccs, reverse=True)
ccs = ccs.select(permutation)
unique_reindexing_operators = flex.select(
unique_reindexing_operators, permutation=permutation)
for i_cb_op, cb_op, cc in zip(count(),
unique_reindexing_operators,
ccs):
combined_cb_op = cache_1.combined_cb_op(other=cache_0,
cb_op=cb_op)
if (not combined_cb_op.c().is_unit_mx()):
reindexing_note = " after reindexing %d using %s" % (
i_0 + 1, combined_cb_op.as_hkl())
else:
reindexing_note = ""
print "CC Obs %d %d %6.3f %s" % (i_1 + 1, i_0 + 1, cc,
combined_cb_op.as_hkl())
print "Correlation of:"
print " %2d:" % (i_1 + 1), cache_1.input.info()
print " %2d:" % (i_0 + 1), cache_0.input.info()
def run(args):
phil = iotbx.phil.process_command_line(args=args,
master_string=master_phil).show()
work_params = phil.work.extract()
log = open(
"%s_%s_merging.log" %
(work_params.output.prefix, work_params.scaling.algorithm), "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
print >> out, "Target unit cell and space group:"
print >> out, " ", work_params.target_unit_cell
print >> out, " ", work_params.target_space_group
miller_set = symmetry(
unit_cell=work_params.target_unit_cell,
space_group_info=work_params.target_space_group).build_miller_set(
anomalous_flag=not work_params.merge_anomalous,
d_min=work_params.d_min)
from xfel.merging.general_fcalc import random_structure
i_model = random_structure(work_params)
# ---- Augment this code with any special procedures for x scaling
scaler = xscaling_manager(miller_set=miller_set,
i_model=i_model,
params=work_params,
log=out)
scaler.read_all_mysql()
print "finished reading the database"
sg = miller_set.space_group()
hkl_asu = scaler.observations["hkl_id"]
imageno = scaler.observations["frame_id"]
intensi = scaler.observations["i"]
lookup = scaler.millers["merged_asu_hkl"]
origH = scaler.observations["H"]
origK = scaler.observations["K"]
origL = scaler.observations["L"]
from cctbx.array_family import flex
print "# observations from the database", len(
scaler.observations["hkl_id"])
hkl = flex.miller_index(flex.select(lookup, hkl_asu))
from cctbx import miller
hkl_list = miller_set.customized_copy(indices=hkl)
ARRAY = miller.array(miller_set=hkl_list, data=intensi)
LATTICES = miller.array(miller_set=hkl_list, data=imageno)
from cctbx.merging.brehm_diederichs import run_multiprocess, run
L = (ARRAY, LATTICES) # tuple(data,lattice_id)
from libtbx import easy_pickle
presort_file = work_params.output.prefix + "_intensities_presort.pickle"
print "pickling these intensities to", presort_file
easy_pickle.dump(presort_file, L)
###### INPUTS #######
# data = miller array: ASU miller index + intensity (sigmas not implemented yet)
# lattice_id = flex double: assignment of each miller index to a lattice number
######################
if work_params.nproc < 5:
print "Sorting the lattices with 1 processor"
result = run(L, nproc=1, verbose=True)
else:
print "Sorting the lattices with %d processors" % work_params.nproc
result = run_multiprocess(L, nproc=work_params.nproc, verbose=False)
for key in result.keys():
print key, len(result[key])
# 2) pickle the postsort (reindexed) ARRAY, LATTICES XXX not done yet; not clear if needed
reverse_lookup = {}
frame_id_list = list(scaler.frames_mysql["frame_id"])
for key in result.keys():
for frame in result[key]:
frame_idx = frame_id_list.index(frame)
reverse_lookup[scaler.frames_mysql["unique_file_name"]
[frame_idx]] = key
lookup_file = work_params.output.prefix + "_lookup.pickle"
reverse_lookup_file = work_params.output.prefix + "_reverse_lookup.pickle"
easy_pickle.dump(lookup_file, result)
easy_pickle.dump(reverse_lookup_file, reverse_lookup)
def run(args):
phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
work_params = phil.work.extract()
log = open("%s_%s_merging.log" % (work_params.output.prefix,work_params.scaling.algorithm), "w")
out = multi_out()
out.register("log", log, atexit_send_to=None)
out.register("stdout", sys.stdout)
print >> out, "Target unit cell and space group:"
print >> out, " ", work_params.target_unit_cell
print >> out, " ", work_params.target_space_group
miller_set = symmetry(
unit_cell=work_params.target_unit_cell,
space_group_info=work_params.target_space_group
).build_miller_set(
anomalous_flag=not work_params.merge_anomalous,
d_min=work_params.d_min)
from xfel.merging.general_fcalc import random_structure
i_model = random_structure(work_params)
# ---- Augment this code with any special procedures for x scaling
scaler = xscaling_manager(
miller_set=miller_set,
i_model=i_model,
params=work_params,
log=out)
scaler.read_all_mysql()
print "finished reading the database"
sg = miller_set.space_group()
hkl_asu = scaler.observations["hkl_id"]
imageno = scaler.observations["frame_id"]
intensi = scaler.observations["i"]
lookup = scaler.millers["merged_asu_hkl"]
origH = scaler.observations["H"]
origK = scaler.observations["K"]
origL = scaler.observations["L"]
from cctbx.array_family import flex
print "# observations from the database",len(scaler.observations["hkl_id"])
hkl = flex.miller_index(flex.select(lookup,hkl_asu))
from cctbx import miller
hkl_list = miller_set.customized_copy(indices = hkl)
ARRAY = miller.array(miller_set = hkl_list, data = intensi)
LATTICES = miller.array(miller_set = hkl_list, data = imageno)
from cctbx.merging.brehm_diederichs import run_multiprocess, run
L = (ARRAY, LATTICES) # tuple(data,lattice_id)
from libtbx import easy_pickle
presort_file = work_params.output.prefix+"_intensities_presort.pickle"
print "pickling these intensities to", presort_file
easy_pickle.dump(presort_file,L)
###### INPUTS #######
# data = miller array: ASU miller index + intensity (sigmas not implemented yet)
# lattice_id = flex double: assignment of each miller index to a lattice number
######################
if work_params.nproc < 5:
print "Sorting the lattices with 1 processor"
result = run(L,nproc=1,verbose=True)
else:
print "Sorting the lattices with %d processors"%work_params.nproc
result = run_multiprocess(L,nproc=work_params.nproc, verbose=False)
for key in result.keys():
print key,len(result[key])
# 2) pickle the postsort (reindexed) ARRAY, LATTICES XXX not done yet; not clear if needed
reverse_lookup = {}
frame_id_list = list(scaler.frames_mysql["frame_id"])
for key in result.keys():
for frame in result[key]:
frame_idx = frame_id_list.index(frame)
reverse_lookup[scaler.frames_mysql["unique_file_name"][frame_idx]] = key
lookup_file = work_params.output.prefix+"_lookup.pickle"
reverse_lookup_file = work_params.output.prefix+"_reverse_lookup.pickle"
easy_pickle.dump(lookup_file, result)
easy_pickle.dump(reverse_lookup_file, reverse_lookup)
def random_elements(size, choices=["O", "Mg", "Si", "Ca"]):
return flex.select(sequence=choices,
permutation=flex.random_size_t(size=size) %
len(choices))
def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) + " [options] directory|file...")
.enable_chunk(easy_all=True)
.enable_multiprocessing()
).process(args=args, min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print "TIME BEGIN cod_refine:", date_and_time()
print
#
master_phil = get_master_phil()
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print
params = work_phil.extract()
#
qi_dict = {}
all_pickles = []
for arg in remaining_args:
if (op.isdir(arg)):
for node in sorted(os.listdir(arg)):
if (node.endswith(".pickle")):
all_pickles.append(op.join(arg, node))
elif (node.startswith("qi_") and len(node) == 10):
qi = open(op.join(arg, node)).read().splitlines()
if (len(qi) == 1):
cod_id = node[3:]
quick_info = eval(qi[0])
assert cod_id not in qi_dict
qi_dict[cod_id] = quick_info
elif (op.isfile(arg)):
all_pickles.append(arg)
else:
raise RuntimeError("Not a file or directory: %s" % arg)
print "Number of pickle files:", len(all_pickles)
print "Number of quick_infos:", len(qi_dict)
sort_choice = params.sorting_of_pickle_files
if (len(qi_dict) != 0 and sort_choice is not None):
print "Sorting pickle files by n_atoms * n_refl:", sort_choice
assert sort_choice in ["down", "up"]
def sort_pickle_files():
if (sort_choice == "down"): i_sign = -1
else: i_sign = 1
buffer = []
for i,path in enumerate(all_pickles):
cod_id = op.basename(path).split(".",1)[0]
qi = qi_dict.get(cod_id)
if (qi is None): nn = 2**31
else: nn = qi[0] * qi[1] * qi[2]
buffer.append((nn, i_sign*i, path))
buffer.sort()
if (i_sign < 0):
buffer.reverse()
result = []
for elem in buffer:
result.append(elem[-1])
return result
all_pickles = sort_pickle_files()
print
#
rss = params.random_subset.size
if (rss is not None and rss > 0):
seed = params.random_subset.seed
print "Selecting subset of %d pickle files using random seed %d" % (
rss, seed)
mt = flex.mersenne_twister(seed=seed)
perm = mt.random_permutation(size=len(all_pickles))[:rss]
flags = flex.bool(len(all_pickles), False).set_selected(perm, True)
all_pickles = flex.select(all_pickles, permutation=flags.iselection())
print
#
from libtbx.path import makedirs_race
if (params.wdir_root is not None):
makedirs_race(path=params.wdir_root)
if (params.pickle_refined_dir is not None):
makedirs_race(path=params.pickle_refined_dir)
#
n_caught = 0
for i_pickle,pickle_file_name in enumerate(all_pickles):
if (i_pickle % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, pickle_file_name)
except KeyboardInterrupt:
print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt"
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s" % pickle_file_name
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
print "done_with: %s (%.2f seconds)" % (pickle_file_name, tm.elapsed())
print
sys.stdout.flush()
print
print "Number of exceptions caught:", n_caught
#
show_times()
print
print "TIME END cod_refine:", date_and_time()
def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) +
" [options] directory|file...").enable_chunk(
easy_all=True).enable_multiprocessing()).process(args=args,
min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print "TIME BEGIN cod_refine:", date_and_time()
print
#
master_phil = get_master_phil()
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print
params = work_phil.extract()
#
qi_dict = {}
all_pickles = []
for arg in remaining_args:
if (op.isdir(arg)):
for node in sorted(os.listdir(arg)):
if (node.endswith(".pickle")):
all_pickles.append(op.join(arg, node))
elif (node.startswith("qi_") and len(node) == 10):
qi = open(op.join(arg, node)).read().splitlines()
if (len(qi) == 1):
cod_id = node[3:]
quick_info = eval(qi[0])
assert cod_id not in qi_dict
qi_dict[cod_id] = quick_info
elif (op.isfile(arg)):
all_pickles.append(arg)
else:
raise RuntimeError("Not a file or directory: %s" % arg)
print "Number of pickle files:", len(all_pickles)
print "Number of quick_infos:", len(qi_dict)
sort_choice = params.sorting_of_pickle_files
if (len(qi_dict) != 0 and sort_choice is not None):
print "Sorting pickle files by n_atoms * n_refl:", sort_choice
assert sort_choice in ["down", "up"]
def sort_pickle_files():
if (sort_choice == "down"): i_sign = -1
else: i_sign = 1
buffer = []
for i, path in enumerate(all_pickles):
cod_id = op.basename(path).split(".", 1)[0]
qi = qi_dict.get(cod_id)
if (qi is None): nn = 2**31
else: nn = qi[0] * qi[1] * qi[2]
buffer.append((nn, i_sign * i, path))
buffer.sort()
if (i_sign < 0):
buffer.reverse()
result = []
for elem in buffer:
result.append(elem[-1])
return result
all_pickles = sort_pickle_files()
print
#
rss = params.random_subset.size
if (rss is not None and rss > 0):
seed = params.random_subset.seed
print "Selecting subset of %d pickle files using random seed %d" % (
rss, seed)
mt = flex.mersenne_twister(seed=seed)
perm = mt.random_permutation(size=len(all_pickles))[:rss]
flags = flex.bool(len(all_pickles), False).set_selected(perm, True)
all_pickles = flex.select(all_pickles, permutation=flags.iselection())
print
#
from libtbx.path import makedirs_race
if (params.wdir_root is not None):
makedirs_race(path=params.wdir_root)
if (params.pickle_refined_dir is not None):
makedirs_race(path=params.pickle_refined_dir)
#
n_caught = 0
for i_pickle, pickle_file_name in enumerate(all_pickles):
if (i_pickle % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, pickle_file_name)
except KeyboardInterrupt:
print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt"
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s" % pickle_file_name
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
print "done_with: %s (%.2f seconds)" % (pickle_file_name,
tm.elapsed())
print
sys.stdout.flush()
print
print "Number of exceptions caught:", n_caught
#
show_times()
print
print "TIME END cod_refine:", date_and_time()
def run(gaussian_fit_pickle_file_names, itvc_file_name, kissel_dir):
itvc_tab = None
if (itvc_file_name is not None):
itvc_tab = itvc_section61_io.read_table6111(itvc_file_name)
fits = read_pickled_fits(gaussian_fit_pickle_file_names)
#easy_pickle.dump("all_fits.pickle", fits)
for k,v in fits.parameters.items():
print "# %s:" % k, v
print
max_errors = flex.double()
labeled_fits = []
n_processed = 0
for label in expected_labels(kissel_dir):
try:
fit_group = fits.all[label]
except Exception:
print "# Warning: Missing scattering_type:", label
else:
print "scattering_type:", label
prev_fit = None
for fit in fit_group:
if (prev_fit is not None):
if (fit.stol > prev_fit.stol):
print "# Warning: decreasing stol"
elif (fit.stol == prev_fit.stol):
if (fit.max_error < prev_fit.max_error):
print "# Warning: same stol but previous has larger error"
prev_fit = fit
fit.sort().show()
gaussian_fit = None
if (itvc_tab is not None and label != "O2-"):
entry = itvc_tab.entries[label]
sel = international_tables_stols <= fit.stol + 1.e-6
gaussian_fit = scitbx.math.gaussian.fit(
international_tables_stols.select(sel),
entry.table_y.select(sel),
entry.table_sigmas.select(sel),
fit)
elif (kissel_dir is not None):
file_name = os.path.join(kissel_dir, "%02d_%s_rf" % (
tiny_pse.table(label).atomic_number(), label))
tab = kissel_io.read_table(file_name)
sel = tab.itvc_sampling_selection() & (tab.x <= fit.stol + 1.e-6)
gaussian_fit = scitbx.math.gaussian.fit(
tab.x.select(sel),
tab.y.select(sel),
tab.sigmas.select(sel),
fit)
if (gaussian_fit is not None):
max_errors.append(
flex.max(gaussian_fit.significant_relative_errors()))
labeled_fits.append(labeled_fit(label, gaussian_fit))
n_processed += 1
print
if (n_processed != len(fits.all)):
print "# Warning: %d fits were not processed." % (
len(fits.all) - n_processed)
print
if (max_errors.size() > 0):
print "Summary:"
perm = flex.sort_permutation(data=max_errors, reverse=True)
max_errors = max_errors.select(perm)
labeled_fits = flex.select(labeled_fits, perm)
quick_summary = {}
for me,lf in zip(max_errors, labeled_fits):
print lf.label, "n_terms=%d max_error: %.4f" % (
lf.gaussian_fit.n_terms(), me)
quick_summary[lf.label + "_" + str(lf.gaussian_fit.n_terms())] = me
if (me > 0.01):
fit = lf.gaussian_fit
re = fit.significant_relative_errors()
for s,y,a,r in zip(fit.table_x(),fit.table_y(),fit.fitted_values(),re):
comment = ""
if (r > 0.01): comment = " large error"
print "%4.2f %7.4f %7.4f %7.4f %7.4f%s" % (s,y,a,a-y,r,comment)
print
print
def run(
file_name,
table_of_gaussians,
cutoff,
low_resolution_only=False,
high_resolution_only=False,
significant_errors_only=False,
plots_dir=None,
quiet=0,
verbose=0,
):
assert not (low_resolution_only and high_resolution_only)
tab = itvc_section61_io.read_table6111(file_name)
for wk in xray_scattering.wk1995_iterator():
label = wk.label()
if not label in tab.entries:
print "Warning: missing scatterer:", label
stols = cctbx.eltbx.gaussian_fit.international_tables_stols
sel = stols <= cutoff + 1.0e-6
stols = stols.select(sel)
if low_resolution_only:
sel = stols <= 2
stols = stols.select(sel)
assert stols.size() == 56
elif high_resolution_only:
sel = stols > 2
stols = stols.select(sel)
assert stols.size() == 6
range_62 = flex.size_t(xrange(62))
labels = flex.std_string()
errors = []
correlations = flex.double()
max_errors = flex.double()
cmp_plots = flex.std_string()
for element in tab.elements:
entry = tab.entries[element]
wk = table_of_gaussians(element, 1)
assert entry.table_y.size() == 62
if not flex.sort_permutation(data=entry.table_y, reverse=True).all_eq(range_62):
print "Increasing: %s (%d)" % (element, entry.atomic_number)
prev_y = entry.table_y[0]
for y in entry.table_y:
if y > prev_y:
print "higher:", y, "before:", prev_y
prev_y = y
raise RuntimeError("Data values are not increasing.")
if low_resolution_only:
gaussian_fit = scitbx.math.gaussian.fit(stols, entry.table_y[:-6], entry.table_sigmas[:-6], wk.fetch())
elif high_resolution_only:
gaussian_fit = scitbx.math.gaussian.fit(stols, entry.table_y[-6:], entry.table_sigmas[-6:], wk.fetch())
elif entry.element != entry.atomic_symbol and entry.table_y[-6:].all_eq(0):
atom_entry = tab.entries[entry.atomic_symbol]
patched_table_y = entry.table_y[:-6]
patched_table_y.append(atom_entry.table_y[-6:])
patched_table_sigmas = entry.table_sigmas[:-6]
patched_table_sigmas.append(atom_entry.table_sigmas[-6:])
gaussian_fit = scitbx.math.gaussian.fit(stols, patched_table_y, patched_table_sigmas, wk.fetch())
else:
gaussian_fit = scitbx.math.gaussian.fit(
stols, entry.table_y[: stols.size()], entry.table_sigmas[: stols.size()], wk.fetch()
)
labels.append(element)
errors.append(gaussian_fit.significant_relative_errors())
max_errors.append(flex.max(errors[-1]))
correlations.append(flex.linear_correlation(gaussian_fit.table_y(), gaussian_fit.fitted_values()).coefficient())
if plots_dir is not None:
if not os.path.isdir(plots_dir):
print "No plots because the directory %s does not exist." % plots_dir
plots_dir = None
else:
cmp_plots.append(
cctbx.eltbx.gaussian_fit.write_plots(plots_dir=plots_dir, label=element, gaussian_fit=gaussian_fit)
)
perm = flex.sort_permutation(data=max_errors, reverse=True)
labels = labels.select(perm)
errors = flex.select(errors, perm)
correlations = correlations.select(perm)
if plots_dir is None:
cmp_plots = [None] * len(labels)
else:
cmp_plots = cmp_plots.select(perm)
for l, e, cc, p in zip(labels, errors, correlations, cmp_plots):
entry = tab.entries[l]
y = entry.table_y
perm = flex.sort_permutation(data=e, reverse=True)[:3]
high = []
for i in perm:
if significant_errors_only and e[i] < 0.01:
break
s = stols[i]
a = ""
if not quiet and s < 2.1:
a = "@%.3f" % y[i]
high.append("%7.4f %4.2f%s" % (e[i], s, a))
if high_resolution_only:
break
if verbose or len(high) > 0:
print "Element %-5s(%2d) cc=%.4f:" % (l, entry.atomic_number, cc), ", ".join(high)
if verbose and p is not None:
print p
sys.stdout.write(open(p).read())
print
def __init__(
self,
intensities,
lattice_ids,
weights=None,
min_pairs=None,
lattice_group=None,
dimensions=None,
nproc=1,
):
r""""Intialise a Target object.
Args:
intensities (cctbx.miller.array): The intensities on which to perform
cosym anaylsis.
lattice_ids (scitbx.array_family.flex.int): An array of equal size to
`intensities` which maps each reflection to a given lattice (dataset).
weights (str): Optionally include weights in the target function.
Allowed values are `None`, "count" and "standard_error". The default
is to use no weights. If "count" is set, then weights are equal to the
number of pairs of reflections used in calculating each value of the
rij matrix. If "standard_error" is used, then weights are defined as
:math:`w_{ij} = 1/s`, where :math:`s = \sqrt{(1-r_{ij}^2)/(n-2)}`.
See also http://www.sjsu.edu/faculty/gerstman/StatPrimer/correlation.pdf.
min_pairs (int): Only calculate the correlation coefficient between two
datasets if they have more than `min_pairs` of common reflections.
lattice_group (cctbx.sgtbx.space_group): Optionally set the lattice
group to be used in the analysis.
dimensions (int): Optionally override the number of dimensions to be used
in the analysis. If not set, then the number of dimensions used is
equal to the greater of 2 or the number of symmetry operations in the
lattice group.
nproc (int): number of processors to use for computing the rij matrix.
"""
if weights is not None:
assert weights in ("count", "standard_error")
self._weights = weights
self._min_pairs = min_pairs
self._nproc = nproc
data = intensities.customized_copy(anomalous_flag=False)
cb_op_to_primitive = data.change_of_basis_op_to_primitive_setting()
data = data.change_basis(cb_op_to_primitive).map_to_asu()
order = flex.sort_permutation(lattice_ids)
sorted_lattice_id = flex.select(lattice_ids, order)
sorted_data = data.data().select(order)
sorted_indices = data.indices().select(order)
self._lattice_ids = sorted_lattice_id
self._data = data.customized_copy(indices=sorted_indices,
data=sorted_data)
assert isinstance(self._data.indices(), type(flex.miller_index()))
assert isinstance(self._data.data(), type(flex.double()))
# construct a lookup for the separate lattices
last_id = -1
self._lattices = flex.int()
for n, lid in enumerate(self._lattice_ids):
if lid != last_id:
last_id = lid
self._lattices.append(n)
self._sym_ops = {"x,y,z"}
self._lattice_group = lattice_group
self._sym_ops.update(
{op.as_xyz()
for op in self._generate_twin_operators()})
if dimensions is None:
dimensions = max(2, len(self._sym_ops))
self.set_dimensions(dimensions)
self._lattice_group = copy.deepcopy(self._data.space_group())
for sym_op in self._sym_ops:
self._lattice_group.expand_smx(sym_op)
self._patterson_group = self._lattice_group.build_derived_patterson_group(
)
logger.debug("Lattice group: %s (%i symops)" %
(self._lattice_group.info().symbol_and_number(),
len(self._lattice_group)))
logger.debug("Patterson group: %s" %
self._patterson_group.info().symbol_and_number())
self._compute_rij_wij()
print " %2d:" % (i_1 + 1), cache_1.input.info()
print " %2d:" % (i_0 + 1), cache_0.input.info()
print "No comparison."
print
else:
ccs = flex.double()
for cb_op in unique_reindexing_operators:
similar_array_0 = cache_0.observations.change_basis(cb_op).map_to_asu()
ccs.append(
cache_1.observations.correlation(
other=similar_array_0, assert_is_similar_symmetry=False
).coefficient()
)
permutation = flex.sort_permutation(ccs, reverse=True)
ccs = ccs.select(permutation)
unique_reindexing_operators = flex.select(unique_reindexing_operators, permutation=permutation)
for i_cb_op, cb_op, cc in zip(count(), unique_reindexing_operators, ccs):
combined_cb_op = cache_1.combined_cb_op(other=cache_0, cb_op=cb_op)
if not combined_cb_op.c().is_unit_mx():
reindexing_note = " after reindexing %d using %s" % (i_0 + 1, combined_cb_op.as_hkl())
else:
reindexing_note = ""
print "CC Obs %d %d %6.3f %s" % (i_1 + 1, i_0 + 1, cc, combined_cb_op.as_hkl())
print "Correlation of:"
print " %2d:" % (i_1 + 1), cache_1.input.info()
print " %2d:" % (i_0 + 1), cache_0.input.info()
print "Overall correlation: %6.3f%s" % (cc, reindexing_note)
show_in_bins = False
if i_cb_op == 0 or (cc >= 0.3 and cc >= ccs[0] - 0.2):
show_in_bins = True
similar_array_0 = cache_0.observations.change_basis(cb_op).map_to_asu()
def run(file_name,
table_of_gaussians,
cutoff,
low_resolution_only=False,
high_resolution_only=False,
significant_errors_only=False,
plots_dir=None,
quiet=0,
verbose=0):
assert not (low_resolution_only and high_resolution_only)
tab = itvc_section61_io.read_table6111(file_name)
for wk in xray_scattering.wk1995_iterator():
label = wk.label()
if (not label in tab.entries):
print("Warning: missing scatterer:", label)
stols = cctbx.eltbx.gaussian_fit.international_tables_stols
sel = stols <= cutoff + 1.e-6
stols = stols.select(sel)
if (low_resolution_only):
sel = stols <= 2
stols = stols.select(sel)
assert stols.size() == 56
elif (high_resolution_only):
sel = stols > 2
stols = stols.select(sel)
assert stols.size() == 6
range_62 = flex.size_t(range(62))
labels = flex.std_string()
errors = []
correlations = flex.double()
max_errors = flex.double()
cmp_plots = flex.std_string()
for element in tab.elements:
entry = tab.entries[element]
wk = table_of_gaussians(element, 1)
assert entry.table_y.size() == 62
if (not flex.sort_permutation(data=entry.table_y,
reverse=True).all_eq(range_62)):
print("Increasing: %s (%d)" % (element, entry.atomic_number))
prev_y = entry.table_y[0]
for y in entry.table_y:
if (y > prev_y):
print("higher:", y, "before:", prev_y)
prev_y = y
raise RuntimeError("Data values are not increasing.")
if (low_resolution_only):
gaussian_fit = scitbx.math.gaussian.fit(stols, entry.table_y[:-6],
entry.table_sigmas[:-6],
wk.fetch())
elif (high_resolution_only):
gaussian_fit = scitbx.math.gaussian.fit(stols, entry.table_y[-6:],
entry.table_sigmas[-6:],
wk.fetch())
elif (entry.element != entry.atomic_symbol
and entry.table_y[-6:].all_eq(0)):
atom_entry = tab.entries[entry.atomic_symbol]
patched_table_y = entry.table_y[:-6]
patched_table_y.append(atom_entry.table_y[-6:])
patched_table_sigmas = entry.table_sigmas[:-6]
patched_table_sigmas.append(atom_entry.table_sigmas[-6:])
gaussian_fit = scitbx.math.gaussian.fit(stols, patched_table_y,
patched_table_sigmas,
wk.fetch())
else:
gaussian_fit = scitbx.math.gaussian.fit(
stols, entry.table_y[:stols.size()],
entry.table_sigmas[:stols.size()], wk.fetch())
labels.append(element)
errors.append(gaussian_fit.significant_relative_errors())
max_errors.append(flex.max(errors[-1]))
correlations.append(
flex.linear_correlation(
gaussian_fit.table_y(),
gaussian_fit.fitted_values()).coefficient())
if (plots_dir is not None):
if (not os.path.isdir(plots_dir)):
print("No plots because the directory %s does not exist." %
plots_dir)
plots_dir = None
else:
cmp_plots.append(
cctbx.eltbx.gaussian_fit.write_plots(
plots_dir=plots_dir,
label=element,
gaussian_fit=gaussian_fit))
perm = flex.sort_permutation(data=max_errors, reverse=True)
labels = labels.select(perm)
errors = flex.select(errors, perm)
correlations = correlations.select(perm)
if (plots_dir is None):
cmp_plots = [None] * len(labels)
else:
cmp_plots = cmp_plots.select(perm)
for l, e, cc, p in zip(labels, errors, correlations, cmp_plots):
entry = tab.entries[l]
y = entry.table_y
perm = flex.sort_permutation(data=e, reverse=True)[:3]
high = []
for i in perm:
if (significant_errors_only and e[i] < 0.01): break
s = stols[i]
a = ""
if (not quiet and s < 2.1): a = "@%.3f" % y[i]
high.append("%7.4f %4.2f%s" % (e[i], s, a))
if (high_resolution_only): break
if (verbose or len(high) > 0):
print("Element %-5s(%2d) cc=%.4f:" % (l, entry.atomic_number, cc),
", ".join(high))
if (verbose and p is not None):
print(p)
sys.stdout.write(open(p).read())
print()
def run(args,
command_name="phenix.reflection_statistics",
additional_miller_arrays=[]):
print("Command line arguments:", end=' ')
for arg in args:
print(arg, end=' ')
print()
print()
command_line = (option_parser(
usage=command_name + " [options] reflection_file [...]",
description="Example: %s data1.mtz data2.sca" % command_name
).enable_symmetry_comprehensive().option(
None,
"--weak_symmetry",
action="store_true",
default=False,
help="symmetry on command line is weaker than symmetry found in files"
).option(
None,
"--quick",
action="store_true",
help="Do not compute statistics between pairs of data arrays"
).enable_resolutions().option(
None,
"--bins",
action="store",
type="int",
dest="n_bins",
default=10,
help="Number of bins",
metavar="INT"
).option(
None,
"--bins_twinning_test",
action="store",
type="int",
dest="n_bins_twinning_test",
default=None,
help="Number of bins for twinning test",
metavar="INT"
).option(
None,
"--bins_second_moments",
action="store",
type="int",
dest="n_bins_second_moments",
default=None,
help="Number of bins for second moments of intensities",
metavar="INT"
).option(
None,
"--lattice_symmetry_max_delta",
action="store",
type="float",
default=3.,
help="angular tolerance in degrees used in the determination"
" of the lattice symmetry"
).option(
None,
"--completeness_as_non_anomalous_or_anomalous",
action="store_true",
help="analyze completeness as is, without conversion to non-anomalous")
).process(args=args)
if (len(command_line.args) == 0 and len(additional_miller_arrays) == 0):
command_line.parser.show_help()
return
active_miller_arrays = []
completeness_as_non_anomalous = (
not command_line.options.completeness_as_non_anomalous_or_anomalous)
n_f_sq_as_f = 0
for file_name in command_line.args:
reflection_file = reflection_file_reader.any_reflection_file(
file_name=file_name)
miller_arrays = None
if (reflection_file.file_type() is not None):
try:
miller_arrays = reflection_file.as_miller_arrays(
crystal_symmetry=command_line.symmetry,
force_symmetry=not command_line.options.weak_symmetry,
merge_equivalents=False)
except Sorry as KeyboardInterrupt:
raise
except Exception:
pass
if (miller_arrays is None):
print("Warning: unknown file format:", file_name, file=sys.stderr)
print(file=sys.stderr)
sys.stderr.flush()
else:
n = _process_miller_arrays(
command_line=command_line,
input_miller_arrays=miller_arrays,
active_miller_arrays=active_miller_arrays)
if (n < 0): return
n_f_sq_as_f += n
if (additional_miller_arrays is not None):
n = _process_miller_arrays(
command_line=command_line,
input_miller_arrays=additional_miller_arrays,
active_miller_arrays=active_miller_arrays)
if (n < 0): return
n_f_sq_as_f += n
if (n_f_sq_as_f > 0):
if (n_f_sq_as_f == 1):
print(
"Note: Intensity array has been converted to an amplitude array."
)
else:
print(
"Note: Intensity arrays have been converted to amplitude arrays."
)
print()
if (len(active_miller_arrays) > 2 and not command_line.options.quick):
print("Array indices (for quick searching):")
for i_0, input_0 in enumerate(active_miller_arrays):
print(" %2d:" % (i_0 + 1), input_0.info())
print()
print("Useful search patterns are:")
print(" Summary i")
print(" CC Obs i j")
print(" CC Ano i j")
print(" i and j are the indices shown above.")
print()
n_bins = command_line.options.n_bins
array_caches = []
for i_0, input_0 in enumerate(active_miller_arrays):
print("Summary", i_0 + 1)
print()
cache_0 = array_cache(
input=input_0,
n_bins=n_bins,
lattice_symmetry_max_delta=\
command_line.options.lattice_symmetry_max_delta,
completeness_as_non_anomalous=completeness_as_non_anomalous)
cache_0.show_possible_twin_laws()
cache_0.show_completeness()
cache_0.show_patterson_peaks()
if (not cache_0.input.space_group().is_centric()):
cache_0.show_perfect_merohedral_twinning_test(
n_bins=command_line.options.n_bins_twinning_test)
else:
cache_0.show_second_moments_of_intensities(
n_bins=command_line.options.n_bins_second_moments)
if (cache_0.input.anomalous_flag()):
print("Anomalous signal of %s:" % str(cache_0.input.info()))
print(cache_0.input.anomalous_signal.__doc__)
anom_signal = cache_0.input.anomalous_signal(use_binning=True)
anom_signal.show()
print()
cache_0.show_measurability()
for i_1, cache_1 in enumerate(array_caches):
unique_reindexing_operators = cache_1.unique_reindexing_operators(
other=cache_0,
relative_length_tolerance=0.05,
absolute_angle_tolerance=5)
if (len(unique_reindexing_operators) == 0):
print("Incompatible unit cells:")
print(" %2d:" % (i_1 + 1), cache_1.input.info())
print(" %2d:" % (i_0 + 1), cache_0.input.info())
print("No comparison.")
print()
else:
ccs = flex.double()
for cb_op in unique_reindexing_operators:
similar_array_0 = cache_0.observations \
.change_basis(cb_op) \
.map_to_asu()
ccs.append(
cache_1.observations.correlation(
other=similar_array_0,
assert_is_similar_symmetry=False).coefficient())
permutation = flex.sort_permutation(ccs, reverse=True)
ccs = ccs.select(permutation)
unique_reindexing_operators = flex.select(
unique_reindexing_operators, permutation=permutation)
for i_cb_op, cb_op, cc in zip(count(),
unique_reindexing_operators,
ccs):
combined_cb_op = cache_1.combined_cb_op(other=cache_0,
cb_op=cb_op)
if (not combined_cb_op.c().is_unit_mx()):
reindexing_note = " after reindexing %d using %s" % (
i_0 + 1, combined_cb_op.as_hkl())
else:
reindexing_note = ""
print("CC Obs %d %d %6.3f %s" %
(i_1 + 1, i_0 + 1, cc, combined_cb_op.as_hkl()))
print("Correlation of:")
print(" %2d:" % (i_1 + 1), cache_1.input.info())
print(" %2d:" % (i_0 + 1), cache_0.input.info())
print("Overall correlation: %6.3f%s" %
(cc, reindexing_note))
show_in_bins = False
if (i_cb_op == 0 or (cc >= 0.3 and cc >= ccs[0] - 0.2)):
show_in_bins = True
similar_array_0 = cache_0.observations \
.change_basis(cb_op) \
.map_to_asu()
cache_1.setup_common_binner(cache_0, n_bins=n_bins)
correlation = cache_1.observations.correlation(
other=similar_array_0,
use_binning=True,
assert_is_similar_symmetry=False)
correlation.show()
print()
if (cache_0.anom_diffs is not None
and cache_1.anom_diffs is not None):
similar_anom_diffs_0 = cache_0.anom_diffs \
.change_basis(cb_op) \
.map_to_asu()
correlation = cache_1.anom_diffs.correlation(
other=similar_anom_diffs_0,
assert_is_similar_symmetry=False)
print("CC Ano %d %d %6.3f %s" %
(i_1 + 1, i_0 + 1, correlation.coefficient(),
combined_cb_op.as_hkl()))
print("Anomalous difference correlation of:")
print(" %2d:" % (i_1 + 1), cache_1.input.info())
print(" %2d:" % (i_0 + 1), cache_0.input.info())
print(
"Overall anomalous difference correlation: %6.3f%s"
% (correlation.coefficient(), reindexing_note))
if (show_in_bins):
correlation = cache_1.anom_diffs.correlation(
other=similar_anom_diffs_0,
use_binning=True,
assert_is_similar_symmetry=False)
correlation.show()
print()
if (not command_line.options.quick):
array_caches.append(cache_0)
print("=" * 79)
print()
def __init__(self, miller_arrays, weights=None, min_pairs=None,
lattice_group=None, dimensions=None, verbose=False,
nproc=1):
self._miller_arrays = miller_arrays
self.verbose = verbose
if weights is not None:
assert weights in ('count', 'standard_error')
self._weights = weights
self._min_pairs = min_pairs
self._nproc = nproc
miller_array_all = None
lattice_ids = None
space_group = None
lattice_id = -1
for intensities in miller_arrays:
assert intensities.is_unique_set_under_symmetry()
lattice_id += 1
if space_group is None:
space_group = intensities.space_group()
else:
assert intensities.space_group() == space_group
ids = intensities.customized_copy(
data=flex.double(intensities.size(), lattice_id), sigmas=None)
assert ids.size() == intensities.size()
if miller_array_all is None:
miller_array_all = intensities
lattice_ids = ids
else:
miller_array_all = miller_array_all.customized_copy(
indices=miller_array_all.indices().concatenate(intensities.indices()),
data=miller_array_all.data().concatenate(intensities.data()),
sigmas=None)
lattice_ids = lattice_ids.customized_copy(
indices=lattice_ids.indices().concatenate(ids.indices()),
data=lattice_ids.data().concatenate(ids.data()))
assert miller_array_all.size() == lattice_ids.size()
data = miller_array_all.customized_copy(anomalous_flag=False)
cb_op_to_primitive = data.change_of_basis_op_to_primitive_setting()
data = data.change_basis(cb_op_to_primitive).map_to_asu()
resort = True
if resort:
order = flex.sort_permutation(lattice_ids.data())
sorted_lattice_id = flex.select(lattice_ids.data(), order)
sorted_data = data.data().select( order)
sorted_indices = data.indices().select( order)
self._lattice_ids = sorted_lattice_id
self._data = data.customized_copy(indices = sorted_indices, data=sorted_data)
else:
self._lattice_ids = self._lattice_ids.data() # type flex int
self._data = data # type miller array with flex double data
assert isinstance(self._data.indices(), type(flex.miller_index()))
assert isinstance(self._data.data(), type(flex.double()))
# construct a lookup for the separate lattices
last_id = -1; self._lattices = flex.int()
for n in xrange(len(self._lattice_ids)):
if self._lattice_ids[n] != last_id:
last_id = self._lattice_ids[n]
self._lattices.append(n)
self._sym_ops = set(['x,y,z'])
self._lattice_group = lattice_group
self._sym_ops.update(
set([op.as_xyz() for op in self.generate_twin_operators()]))
if dimensions is None:
dimensions = max(2, len(self._sym_ops))
self.set_dimensions(dimensions)
import copy
self._lattice_group = copy.deepcopy(self._data.space_group())
for sym_op in self._sym_ops:
self._lattice_group.expand_smx(sym_op)
self._patterson_group = self._lattice_group.build_derived_patterson_group()
logger.debug(
'Lattice group: %s (%i symops)' %(
self._lattice_group.info().symbol_and_number(), len(self._lattice_group)))
logger.debug(
'Patterson group: %s' %self._patterson_group.info().symbol_and_number())
import time
t0 = time.time()
self.compute_rij_wij()
t1 = time.time()
import scitbx.math
rij = self.rij_matrix.as_1d()
non_zero_sel = rij != 0
logger.debug('Computed Rij matrix in %.2f seconds' %(t1 - t0))
logger.debug('%i (%.1f%%) non-zero elements of Rij matrix' %(
non_zero_sel.count(True), 100*non_zero_sel.count(True)/non_zero_sel.size()))
scitbx.math.basic_statistics(rij.select(non_zero_sel)).show(f=debug_handle)
return
