def sample_e_pot(id_code, f_obs, xray_structure, edge_list, params):
if (edge_list is None):
print "NO_TARDY: no edge_list"
return
#
xs = xray_structure
if (xs.special_position_indices().size() != 0):
print "NO_TARDY: special positions"
return
sites_cart = xs.sites_cart()
labels = xs.scatterers().extract_labels()
pat = xs.pair_asu_table(distance_cutoff=2.5)
if (0):
pat.show_distances(sites_cart=sites_cart, site_labels=labels)
pst = pat.extract_pair_sym_table()
for i,j in edge_list:
assert i <= j
sym_dict = pst[i].get(j)
if (sym_dict is None):
print "NO_TARDY: large distance for edge:", labels[i], labels[j]
return
#
import scitbx.graph.tardy_tree
tt = scitbx.graph.tardy_tree.construct(
sites=sites_cart, edge_list=edge_list)
import scitbx.rigid_body
tardy_model = scitbx.rigid_body.tardy_model(
labels=labels,
sites=sites_cart,
masses=xs.atomic_weights(),
tardy_tree=tt,
potential_obj=potential_object(f_obs, xs),
near_singular_hinges_angular_tolerance_deg=5)
if (tardy_model.number_of_trees != 1):
print "NO_TARDY: multiple trees"
return
#
print "Single tardy tree:", \
id_code, xs.scatterers().size(), xs.space_group_info()
tt.show_summary(vertex_labels=labels, prefix=" ")
print "dof each joint:", list(tardy_model.degrees_of_freedom_each_joint())
print "q_size each joint:", list(tardy_model.q_size_each_joint())
q_packed = tardy_model.pack_q()
print "q_packed.size():", q_packed.size()
print "q_packed:", numstr(q_packed)
if (params.iq < 0):
return
assert params.iq < q_packed.size()
#
xy = []
from libtbx.utils import xsamples
from math import pi
for q_deg in xsamples(params.qmin, params.qmax, params.qstep):
q_packed[params.iq] = q_deg*pi/180
tardy_model.unpack_q(q_packed=q_packed)
e_pot = tardy_model.e_pot()
xy.append((q_deg,e_pot))
from libtbx import pyplot
pyplot.plot_pairs(xy, "r-")
pyplot.show()
def sample_e_pot(id_code, f_obs, xray_structure, edge_list, params):
if (edge_list is None):
print("NO_TARDY: no edge_list")
return
#
xs = xray_structure
if (xs.special_position_indices().size() != 0):
print("NO_TARDY: special positions")
return
sites_cart = xs.sites_cart()
labels = xs.scatterers().extract_labels()
pat = xs.pair_asu_table(distance_cutoff=2.5)
if (0):
pat.show_distances(sites_cart=sites_cart, site_labels=labels)
pst = pat.extract_pair_sym_table()
for i, j in edge_list:
assert i <= j
sym_dict = pst[i].get(j)
if (sym_dict is None):
print("NO_TARDY: large distance for edge:", labels[i], labels[j])
return
#
import scitbx.graph.tardy_tree
tt = scitbx.graph.tardy_tree.construct(sites=sites_cart,
edge_list=edge_list)
import scitbx.rigid_body
tardy_model = scitbx.rigid_body.tardy_model(
labels=labels,
sites=sites_cart,
masses=xs.atomic_weights(),
tardy_tree=tt,
potential_obj=potential_object(f_obs, xs),
near_singular_hinges_angular_tolerance_deg=5)
if (tardy_model.number_of_trees != 1):
print("NO_TARDY: multiple trees")
return
#
print("Single tardy tree:", \
id_code, xs.scatterers().size(), xs.space_group_info())
tt.show_summary(vertex_labels=labels, prefix=" ")
print("dof each joint:", list(tardy_model.degrees_of_freedom_each_joint()))
print("q_size each joint:", list(tardy_model.q_size_each_joint()))
q_packed = tardy_model.pack_q()
print("q_packed.size():", q_packed.size())
print("q_packed:", numstr(q_packed))
if (params.iq < 0):
return
assert params.iq < q_packed.size()
#
xy = []
from libtbx.utils import xsamples
from math import pi
for q_deg in xsamples(params.qmin, params.qmax, params.qstep):
q_packed[params.iq] = q_deg * pi / 180
tardy_model.unpack_q(q_packed=q_packed)
e_pot = tardy_model.e_pot()
xy.append((q_deg, e_pot))
from libtbx import pyplot
pyplot.plot_pairs(xy, "r-")
pyplot.show()
def f_obs_and_f_calc_agree_well(O, co):
if (O.c_obs.indices().size() == 0): return False
from cctbx.array_family import flex
f_obs = O.c_obs.as_amplitude_array(algorithm="xtal_3_7")
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure=O.xray_structure).f_calc().amplitudes()
fan_out_sel = f_obs.f_obs_f_calc_fan_outlier_selection(
f_calc=f_calc,
offset_low=co.fan_offset_low,
offset_high=co.fan_offset_high,
also_return_x_and_y=True)
if (fan_out_sel is None):
return False
if (co.i_obs_i_calc_plot and f_obs.indices().size() != 0):
from libtbx import pyplot
xs = O.c_obs.as_intensity_array(algorithm="simple").data()
ys = flex.pow2(f_calc.data())
pyplot.plot(xs.as_numpy_array(), ys.as_numpy_array(), "ro")
pyplot.show()
fan_out_sel, x, y = fan_out_sel
fan_in_sel = ~fan_out_sel
if (co.f_obs_f_calc_plot):
from libtbx import pyplot
xs = x.select(fan_out_sel)
ys = y.select(fan_out_sel)
if (xs.size() == 0):
pyplot.plot(x.as_numpy_array(), y.as_numpy_array(), "bo")
else:
pyplot.plot(xs.as_numpy_array(), ys.as_numpy_array(), "ro")
xs = x.select(fan_in_sel)
ys = y.select(fan_in_sel)
if (xs.size() != 0):
pyplot.plot(xs.as_numpy_array(), ys.as_numpy_array(), "bo")
pyplot.plot_pairs([(co.fan_offset_low, 0),
(1, 1 - co.fan_offset_high)], "r-")
pyplot.plot_pairs([(0, co.fan_offset_low),
(1 - co.fan_offset_high, 1)], "r-")
pyplot.plot_pairs([(0, 0), (1, 1)], "k--")
pyplot.show()
fan_outlier_fraction = fan_out_sel.count(True) / fan_out_sel.size()
def cc_r1(fo, fc):
lc = flex.linear_correlation(fo.data(), fc.data())
assert lc.is_well_defined()
cc = lc.coefficient()
from libtbx import Auto
r1 = f_obs.r1_factor(other=f_calc, scale_factor=Auto)
return cc, r1
cc_all, r1_all = cc_r1(f_obs, f_calc)
cc_in, r1_in = cc_r1(f_obs.select(fan_in_sel),
f_calc.select(fan_in_sel))
print "f_obs_f_calc %s" % O.cod_id, \
"| cc_all %.4f | r1_all %.4f | out %.4f | cc_in %.4f | r1_in %.4f |" % (
cc_all, r1_all, fan_outlier_fraction, cc_in, r1_in)
if (fan_outlier_fraction > co.max_fan_outlier_fraction):
return False
if (cc_all < co.min_f_obs_f_calc_correlation):
return False
return True
def report(O, plot=None, xy_prefix=None):
from cctbx.array_family import flex
print "Number of shots:", O.completeness_history.size() - 1
print
print "Histogram of counts per reflection:"
flex.histogram(O.counts.as_double(),
n_slots=8).show(prefix=" ", format_cutoffs="%7.0f")
print
print "Observations per reflection:"
flex.show_count_stats(counts=O.counts, prefix=" ")
print " Median:", int(flex.median(O.counts.as_double()) + 0.5)
print
sys.stdout.flush()
if (xy_prefix is None):
xy_prefix = ""
elif (len(xy_prefix) != 0):
xy_prefix = xy_prefix + "_"
def dump_xy(name, array):
f = open(xy_prefix + "%s.xy" % name, "w")
for i, c in enumerate(array):
print >> f, i, c
dump_xy("completeness_history", O.completeness_history)
dump_xy("min_count_history", O.min_count_history)
if (O.use_symmetry): _ = O.i_calc.asu
else: _ = O.i_calc.p1_anom
_ = _.customized_copy(data=O.counts).sort(by_value="resolution")
sym_factors = _.space_group().order_p()
if (not O.i_calc.asu.anomalous_flag()):
sym_factors *= 2
sym_factors /= _.multiplicities().data()
counts_sorted_by_resolution = _.data().as_int() * sym_factors
dump_xy("counts_sorted_by_resolution", counts_sorted_by_resolution)
dump_xy("d_spacings_sorted_by_resolution", _.d_spacings().data())
if (plot == "completeness"):
from libtbx import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
_ = O.completeness_history
nx = _.size()
ax.plot(range(nx), _, "r-")
ax.axis([0, nx, 0, 1])
pyplot.show()
elif (plot == "redundancy"):
from libtbx import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
_ = counts_sorted_by_resolution
ax.plot(range(len(_)), _, "r-")
ax.axis([-_.size() * 0.05, _.size() * 1.05, 0, None])
pyplot.show()
elif (plot is not None):
raise RuntimeError('Unknown plot type: "%s"' % plot)
def report(O, plot=None, xy_prefix=None):
from cctbx.array_family import flex
print "Number of shots:", O.completeness_history.size()-1
print
print "Histogram of counts per reflection:"
flex.histogram(O.counts.as_double(), n_slots=8).show(
prefix=" ", format_cutoffs="%7.0f")
print
print "Observations per reflection:"
flex.show_count_stats(counts=O.counts, prefix=" ")
print " Median:", int(flex.median(O.counts.as_double())+0.5)
print
sys.stdout.flush()
if (xy_prefix is None):
xy_prefix = ""
elif (len(xy_prefix) != 0):
xy_prefix = xy_prefix + "_"
def dump_xy(name, array):
f = open(xy_prefix + "%s.xy" % name, "w")
for i,c in enumerate(array):
print >> f, i, c
dump_xy("completeness_history", O.completeness_history)
dump_xy("min_count_history", O.min_count_history)
if (O.use_symmetry): _ = O.i_calc.asu
else: _ = O.i_calc.p1_anom
_ = _.customized_copy(data=O.counts).sort(by_value="resolution")
sym_factors = _.space_group().order_p()
if (not O.i_calc.asu.anomalous_flag()):
sym_factors *= 2
sym_factors /= _.multiplicities().data()
counts_sorted_by_resolution = _.data().as_int() * sym_factors
dump_xy("counts_sorted_by_resolution", counts_sorted_by_resolution)
dump_xy("d_spacings_sorted_by_resolution", _.d_spacings().data())
if (plot == "completeness"):
from libtbx import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
_ = O.completeness_history
nx = _.size()
ax.plot(range(nx), _, "r-")
ax.axis([0, nx, 0, 1])
pyplot.show()
elif (plot == "redundancy"):
from libtbx import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
_ = counts_sorted_by_resolution
ax.plot(range(len(_)), _, "r-")
ax.axis([-_.size()*0.05, _.size()*1.05, 0, None])
pyplot.show()
elif (plot is not None):
raise RuntimeError('Unknown plot type: "%s"' % plot)
def plot_samples_ix(O, stage, ix):
p = O.params.plot_samples
i_sc, x_type = O.x_info[ix]
def f_calc_without_moving_scatterer():
sc = O.xray_structure.scatterers()[i_sc]
occ = sc.occupancy
try:
sc.occupancy = 0
result = O.get_f_calc().data()
finally:
sc.occupancy = occ
return result
f_calc_fixed = f_calc_without_moving_scatterer()
xs = O.xray_structure.select(flex.size_t([i_sc]))
def f_calc_moving():
return O.f_obs.structure_factors_from_scatterers(
xray_structure=xs,
algorithm="direct",
cos_sin_table=False).f_calc().data()
sc = xs.scatterers()[0]
ss = xs.site_symmetry_table().get(0)
def build_info_str():
return "%s|%03d|%03d|%s|%s|occ=%.2f|%s|%s" % (
O.plot_samples_id,
ix,
i_sc,
sc.label,
sc.scattering_type,
sc.occupancy,
ss.special_op_simplified(),
x_type)
info_str = build_info_str()
print "plot_samples:", info_str
sys.stdout.flush()
ys = []
def ys_append():
tg = O.__get_tg(
f_cbs=O.get_f_cbs(
f_calc=O.f_obs.customized_copy(data=f_calc_moving()+f_calc_fixed)),
derivatives_depth=0,
target=O.ps_target,
weights=O.ps_weights)
y = tg.target_work()
ys.append(y)
return y
xyv = []
if (x_type == "u"):
assert p.u_min < p.u_max
assert p.u_steps > 0
for i_step in xrange(p.u_steps+1):
u = p.u_min + i_step / p.u_steps * (p.u_max - p.u_min)
sc.u_iso = u
y = ys_append()
xyv.append((u,y,sc.u_iso))
else:
assert p.x_radius > 0
assert p.x_steps > 0
ss_constr = ss.site_constraints()
ss_np = ss_constr.n_independent_params()
ss_ip = "xyz".find(x_type)
assert ss_ip >= 0
ixx = ix - ss_ip
indep = list(O.x[ixx:ixx+ss_np])
i_inp = indep[ss_ip]
from libtbx.test_utils import approx_equal
assert approx_equal(
ss_constr.all_params(independent_params=indep), sc.site)
site_inp = sc.site
indep[ss_ip] = i_inp + 1
sc.site = ss_constr.all_params(independent_params=indep)
dist = xs.unit_cell().distance(sc.site, site_inp)
assert dist != 0
x_scale = p.x_radius / dist
for i_step in xrange(-p.x_steps//2, p.x_steps//2+1):
x = i_step / p.x_steps * 2 * x_scale
indep[ss_ip] = i_inp + x
sc.site = ss_constr.all_params(independent_params=indep)
y = ys_append()
dist = xs.unit_cell().distance(sc.site, site_inp)
if (i_step < 0): dist *= -1
xyv.append((dist,y,indep[ss_ip]))
#
base_name_plot_files = "%s_%03d_%s" % (p.file_prefix, ix, stage)
def write_xyv():
if (p.file_prefix is None): return
f = open(base_name_plot_files+".xy", "w")
print >> f, "# %s" % info_str
print >> f, "# %s" % str(xs.unit_cell())
print >> f, "# %s" % str(xs.space_group_info().symbol_and_number())
for x,y,v in xyv:
print >> f, x, y, v
write_xyv()
if (len(p.pyplot) != 0):
from libtbx import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
x,y,v = zip(*xyv)
ax.plot(x,y, "r-")
x = O.x[ix]
ax.plot([x, x], [min(ys), max(ys)], "k--")
if (O.x_reference is not None):
x = O.x_reference[ix]
ax.plot([x, x], [min(ys), max(ys)], "r--")
ax.set_title(info_str, fontsize=12)
ax.axis([xyv[0][0], xyv[-1][0], None, None])
def write_pdf():
if (p.file_prefix is None): return
fig.savefig(base_name_plot_files+".pdf", bbox_inches="tight")
if ("pdf" in p.pyplot):
write_pdf()
if ("gui" in p.pyplot):
pyplot.show()
def plot_samples_ix(O, stage, ix):
p = O.params.plot_samples
i_sc, x_type = O.x_info[ix]
def f_calc_without_moving_scatterer():
sc = O.xray_structure.scatterers()[i_sc]
occ = sc.occupancy
try:
sc.occupancy = 0
result = O.get_f_calc().data()
finally:
sc.occupancy = occ
return result
f_calc_fixed = f_calc_without_moving_scatterer()
xs = O.xray_structure.select(flex.size_t([i_sc]))
def f_calc_moving():
return O.f_obs.structure_factors_from_scatterers(
xray_structure=xs,
algorithm="direct",
cos_sin_table=False).f_calc().data()
sc = xs.scatterers()[0]
ss = xs.site_symmetry_table().get(0)
def build_info_str():
return "%s|%03d|%03d|%s|%s|occ=%.2f|%s|%s" % (
O.plot_samples_id,
ix,
i_sc,
sc.label,
sc.scattering_type,
sc.occupancy,
ss.special_op_simplified(),
x_type)
info_str = build_info_str()
print "plot_samples:", info_str
sys.stdout.flush()
ys = []
def ys_append():
tg = O.__get_tg(
f_cbs=O.get_f_cbs(
f_calc=O.f_obs.customized_copy(data=f_calc_moving()+f_calc_fixed)),
derivatives_depth=0,
target=O.ps_target,
weights=O.ps_weights)
y = tg.target_work()
ys.append(y)
return y
xyv = []
if (x_type == "u"):
assert p.u_min < p.u_max
assert p.u_steps > 0
for i_step in xrange(p.u_steps+1):
u = p.u_min + i_step / p.u_steps * (p.u_max - p.u_min)
sc.u_iso = u
y = ys_append()
xyv.append((u,y,sc.u_iso))
else:
assert p.x_radius > 0
assert p.x_steps > 0
ss_constr = ss.site_constraints()
ss_np = ss_constr.n_independent_params()
ss_ip = "xyz".find(x_type)
assert ss_ip >= 0
ixx = ix - ss_ip
indep = list(O.x[ixx:ixx+ss_np])
i_inp = indep[ss_ip]
from libtbx.test_utils import approx_equal
assert approx_equal(
ss_constr.all_params(independent_params=indep), sc.site)
site_inp = sc.site
indep[ss_ip] = i_inp + 1
sc.site = ss_constr.all_params(independent_params=indep)
dist = xs.unit_cell().distance(sc.site, site_inp)
assert dist != 0
x_scale = p.x_radius / dist
for i_step in xrange(-p.x_steps//2, p.x_steps//2+1):
x = i_step / p.x_steps * 2 * x_scale
indep[ss_ip] = i_inp + x
sc.site = ss_constr.all_params(independent_params=indep)
y = ys_append()
dist = xs.unit_cell().distance(sc.site, site_inp)
if (i_step < 0): dist *= -1
xyv.append((dist,y,indep[ss_ip]))
#
base_name_plot_files = "%s_%03d_%s" % (p.file_prefix, ix, stage)
def write_xyv():
if (p.file_prefix is None): return
f = open(base_name_plot_files+".xy", "w")
print >> f, "# %s" % info_str
print >> f, "# %s" % str(xs.unit_cell())
print >> f, "# %s" % str(xs.space_group_info().symbol_and_number())
for x,y,v in xyv:
print >> f, x, y, v
write_xyv()
if (len(p.pyplot) != 0):
from libtbx import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
x,y,v = zip(*xyv)
ax.plot(x,y, "r-")
x = O.x[ix]
ax.plot([x, x], [min(ys), max(ys)], "k--")
if (O.x_reference is not None):
x = O.x_reference[ix]
ax.plot([x, x], [min(ys), max(ys)], "r--")
ax.set_title(info_str, fontsize=12)
ax.axis([xyv[0][0], xyv[-1][0], None, None])
def write_pdf():
if (p.file_prefix is None): return
fig.savefig(base_name_plot_files+".pdf", bbox_inches="tight")
if ("pdf" in p.pyplot):
write_pdf()
if ("gui" in p.pyplot):
pyplot.show()
def f_obs_and_f_calc_agree_well(O, co):
if O.c_obs.indices().size() == 0:
return False
from cctbx.array_family import flex
f_obs = O.c_obs.as_amplitude_array(algorithm="xtal_3_7")
f_calc = f_obs.structure_factors_from_scatterers(xray_structure=O.xray_structure).f_calc().amplitudes()
fan_out_sel = f_obs.f_obs_f_calc_fan_outlier_selection(
f_calc=f_calc, offset_low=co.fan_offset_low, offset_high=co.fan_offset_high, also_return_x_and_y=True
)
if fan_out_sel is None:
return False
if co.i_obs_i_calc_plot and f_obs.indices().size() != 0:
from libtbx import pyplot
xs = O.c_obs.as_intensity_array(algorithm="simple").data()
ys = flex.pow2(f_calc.data())
pyplot.plot(xs.as_numpy_array(), ys.as_numpy_array(), "ro")
pyplot.show()
fan_out_sel, x, y = fan_out_sel
fan_in_sel = ~fan_out_sel
if co.f_obs_f_calc_plot:
from libtbx import pyplot
xs = x.select(fan_out_sel)
ys = y.select(fan_out_sel)
if xs.size() == 0:
pyplot.plot(x.as_numpy_array(), y.as_numpy_array(), "bo")
else:
pyplot.plot(xs.as_numpy_array(), ys.as_numpy_array(), "ro")
xs = x.select(fan_in_sel)
ys = y.select(fan_in_sel)
if xs.size() != 0:
pyplot.plot(xs.as_numpy_array(), ys.as_numpy_array(), "bo")
pyplot.plot_pairs([(co.fan_offset_low, 0), (1, 1 - co.fan_offset_high)], "r-")
pyplot.plot_pairs([(0, co.fan_offset_low), (1 - co.fan_offset_high, 1)], "r-")
pyplot.plot_pairs([(0, 0), (1, 1)], "k--")
pyplot.show()
fan_outlier_fraction = fan_out_sel.count(True) / fan_out_sel.size()
def cc_r1(fo, fc):
lc = flex.linear_correlation(fo.data(), fc.data())
assert lc.is_well_defined()
cc = lc.coefficient()
from libtbx import Auto
r1 = f_obs.r1_factor(other=f_calc, scale_factor=Auto)
return cc, r1
cc_all, r1_all = cc_r1(f_obs, f_calc)
cc_in, r1_in = cc_r1(f_obs.select(fan_in_sel), f_calc.select(fan_in_sel))
print "f_obs_f_calc %s" % O.cod_id, "| cc_all %.4f | r1_all %.4f | out %.4f | cc_in %.4f | r1_in %.4f |" % (
cc_all,
r1_all,
fan_outlier_fraction,
cc_in,
r1_in,
)
if fan_outlier_fraction > co.max_fan_outlier_fraction:
return False
if cc_all < co.min_f_obs_f_calc_correlation:
return False
return True