def run(args):
import libtbx.load_env
usage = "%s experiments.json indexed.pickle [options]" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0:
parser.print_help()
return
elif len(experiments) > 1:
raise Sorry("More than one experiment present")
experiment = experiments[0]
assert(len(reflections) == 1)
reflections = reflections[0]
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
if 'intensity.prf.value' in reflections:
intensities = reflections['intensity.prf.value']
variances = reflections['intensity.prf.variance']
sel = (variances > 0)
intensities = intensities.select(sel)
variances = variances.select(sel)
sigmas = flex.sqrt(variances)
indices = reflections['miller_index'].select(sel)
from cctbx import crystal, miller
crystal_symmetry = crystal.symmetry(
space_group=experiment.crystal.get_space_group(),
unit_cell=experiment.crystal.get_unit_cell())
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
anomalous_flag=True,
indices=indices)
miller_array = miller.array(
miller_set=miller_set,
data=intensities,
sigmas=sigmas).set_observation_type_xray_intensity()
#miller_array.setup_binner(n_bins=50, reflections_per_bin=100)
miller_array.setup_binner(auto_binning=True, n_bins=20)
result = miller_array.i_over_sig_i(use_binning=True)
result.show()
from cctbx import uctbx
d_star_sq_centre = result.binner.bin_centers(2)
i_over_sig_i = flex.double(
[d if d is not None else 0 for d in result.data[1:-1]])
sel = (i_over_sig_i > 0)
d_star_sq_centre = d_star_sq_centre.select(sel)
i_over_sig_i = i_over_sig_i.select(sel)
log_i_over_sig_i = flex.log(i_over_sig_i)
weights = result.binner.counts()[1:-1].as_double().select(sel)
fit = flex.linear_regression(
d_star_sq_centre, log_i_over_sig_i, weights=weights)
m = fit.slope()
c = fit.y_intercept()
import math
y_cutoff = math.log(params.i_sigi_cutoff)
x_cutoff = (y_cutoff - c)/m
estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff)
print "estimated d_min: %.2f" %estimated_d_min
if params.plot:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.plot(
list(d_star_sq_centre),
list(log_i_over_sig_i),
label=r"ln(I/sigI)")
ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()], color='red')
ax.plot([x_cutoff, x_cutoff], pyplot.ylim(), color='grey', linestyle='dashed')
ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff], color='grey', linestyle='dashed')
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(I/sigI)")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
pyplot.savefig("estimate_resolution_limit.png")
pyplot.clf()
def __call__(self):
"""Determine optimal mosaicity and domain size model (monochromatic)"""
if self.refinery is None:
RR = self.reflections
else:
RR = self.refinery.predict_for_reflection_table(self.reflections)
all_crystals = []
self.nv_acceptance_flags = flex.bool(len(self.reflections["id"]))
from dxtbx.model import MosaicCrystalSauter2014
for iid, experiment in enumerate(self.experiments):
excursion_rad = RR["delpsical.rad"].select(RR["id"] == iid)
delta_psi_deg = excursion_rad * 180.0 / math.pi
logger.info("")
logger.info("%s %s", flex.max(delta_psi_deg),
flex.min(delta_psi_deg))
mean_excursion = flex.mean(delta_psi_deg)
logger.info(
"The mean excursion is %7.3f degrees, r.m.s.d %7.3f",
mean_excursion,
math.sqrt(flex.mean(RR["delpsical2"].select(RR["id"] == iid))),
)
crystal = MosaicCrystalSauter2014(self.experiments[iid].crystal)
self.experiments[iid].crystal = crystal
beam = self.experiments[iid].beam
miller_indices = self.reflections["miller_index"].select(
self.reflections["id"] == iid)
# FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection
two_thetas = crystal.get_unit_cell().two_theta(
miller_indices, beam.get_wavelength(), deg=True)
dspacings = crystal.get_unit_cell().d(miller_indices)
# First -- try to get a reasonable envelope for the observed excursions.
# minimum of three regions; maximum of 50 measurements in each bin
logger.info("fitting parameters on %d spots", len(excursion_rad))
n_bins = min(max(3, len(excursion_rad) // 25), 50)
bin_sz = len(excursion_rad) // n_bins
logger.info("nbins %s bin_sz %s", n_bins, bin_sz)
order = flex.sort_permutation(two_thetas)
two_thetas_env = flex.double()
dspacings_env = flex.double()
excursion_rads_env = flex.double()
for x in range(0, n_bins):
subset = order[x * bin_sz:(x + 1) * bin_sz]
two_thetas_env.append(flex.mean(two_thetas.select(subset)))
dspacings_env.append(flex.mean(dspacings.select(subset)))
excursion_rads_env.append(
flex.max(flex.abs(excursion_rad.select(subset))))
# Second -- parameter fit
# solve the normal equations
sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
sum_inv_u = flex.sum(dspacings_env)
sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
sum_te = flex.sum(excursion_rads_env)
Normal_Mat = sqr(
(sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
Vector = col((sum_te_u, sum_te))
solution = Normal_Mat.inverse() * Vector
s_ang = 1.0 / (2 * solution[0])
logger.info("Best LSQ fit Scheerer domain size is %9.2f ang",
s_ang)
k_degrees = solution[1] * 180.0 / math.pi
logger.info(
"The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f",
2 * k_degrees,
k_degrees,
)
from xfel.mono_simulation.max_like import minimizer
# coerce the estimates to be positive for max-likelihood
lower_limit_domain_size = (
math.pow(crystal.get_unit_cell().volume(), 1.0 / 3.0) * 3
) # params.refinement.domain_size_lower_limit
d_estimate = max(s_ang, lower_limit_domain_size)
M = minimizer(
d_i=dspacings,
psi_i=excursion_rad,
eta_rad=abs(2.0 * solution[1]),
Deff=d_estimate,
)
logger.info(
"ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots",
M.x[1] * 180.0 / math.pi,
2.0 / M.x[0],
len(two_thetas),
)
tan_phi_rad_ML = dspacings / (2.0 / M.x[0])
tan_phi_deg_ML = tan_phi_rad_ML * 180.0 / math.pi
tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180.0 / math.pi
# Only set the flags for those reflections that were indexed for this lattice
self.nv_acceptance_flags.set_selected(
self.reflections["id"] == iid,
flex.abs(delta_psi_deg) < tan_outer_deg_ML,
)
if (
self.graph_verbose
): # params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit
AD1TF7B_MAX2T = 30.0
AD1TF7B_MAXDP = 1.0
from matplotlib import pyplot as plt
plt.plot(two_thetas, delta_psi_deg, "bo")
minplot = flex.min(two_thetas)
plt.plot([0, minplot], [mean_excursion, mean_excursion], "k-")
LR = flex.linear_regression(two_thetas, delta_psi_deg)
model_y = LR.slope() * two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
plt.title(
"ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" %
(M.x[1] * 180.0 / math.pi, 2.0 / M.x[0], len(two_thetas)))
plt.plot(two_thetas, tan_phi_deg_ML, "r.")
plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
plt.plot(two_thetas, tan_outer_deg_ML, "g.")
plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
plt.xlim([0, AD1TF7B_MAX2T])
plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP])
plt.show()
plt.close()
from xfel.mono_simulation.util import green_curve_area
self.green_curve_area = green_curve_area(two_thetas,
tan_outer_deg_ML)
logger.info("The green curve area is %s", self.green_curve_area)
crystal.set_half_mosaicity_deg(M.x[1] * 180.0 / (2.0 * math.pi))
crystal.set_domain_size_ang(2.0 / M.x[0])
self._ML_full_mosaicity_rad = M.x[1]
self._ML_domain_size_ang = 2.0 / M.x[0]
# params.refinement.mosaic.model_expansion_factor
"""The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes
as close as possible to # of observed reflections input, in the last integration call. Determine this by
inspecting the output log file interactively. Do not exceed the bare minimum threshold needed.
The intention is to find an optimal value, global for a given dataset."""
model_expansion_factor = 1.4
crystal.set_half_mosaicity_deg(crystal.get_half_mosaicity_deg() *
model_expansion_factor)
crystal.set_domain_size_ang(crystal.get_domain_size_ang() /
model_expansion_factor)
if (self.ewald_proximal_volume(iid) >
self.params.indexing.stills.ewald_proximal_volume_max):
raise DialsIndexError("Ewald proximity volume too high, %f" %
self.ewald_proximal_volume(iid))
all_crystals.append(crystal)
return all_crystals
def estimate_resolution_limit(reflections, ice_sel=None, plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections["intensity.sum.value"]
variances = reflections["intensity.sum.variance"]
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities / flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(d_star_sq.select(~ice_sel),
log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(d_star_sq,
target_n_per_bin=20,
max_slots=20,
min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1 - low_percentile_limit
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
if sel.count(True) == 0:
continue
outliers = wilson_outliers(reflections.select(sel_all),
ice_sel=ice_sel.select(sel_all))
outliers_all.set_selected(sel_all, outliers)
isel = sel_all.iselection().select(~(outliers)
& ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
if m_upper == m_lower:
intersection = (-1, -1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
# with:
# a_ = m_upper
# b_ = m_lower
# c_ = c_upper
# d_ = c_lower
# intersection == ((d_ - c_) / (a_ - b_), (a_ * d_ - b_ * c_) / (a_ - b_))
intersection = (
(c_lower - c_upper) / (m_upper - m_lower),
(m_upper * c_lower - m_lower * c_upper) / (m_upper - m_lower),
)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper,
c_upper)
inside = inside & ~outliers_all & ~ice_sel
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
if plot_filename is not None:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(d_star_sq, log_i_over_sigi, marker="+")
ax.scatter(
d_star_sq.select(inside),
log_i_over_sigi.select(inside),
marker="+",
color="green",
)
ax.scatter(
d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker="+",
color="black",
)
ax.scatter(
d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker="+",
color="grey",
)
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker="+", color="red")
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker="+", color="red")
if intersection[0] <= ax.get_xlim(
)[1] and intersection[1] <= ax.get_ylim()[1]:
ax.scatter([intersection[0]], [intersection[1]],
marker="x",
s=50,
color="b")
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color="red")
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color="red")
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(I/sigI)")
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(
uctbx.d_as_d_star_sq(slot.d_max),
0,
ax.get_ylim()[1],
linestyle="dotted",
color="grey",
)
ax.vlines(
uctbx.d_as_d_star_sq(slot.d_min),
0,
ax.get_ylim()[1],
linestyle="dotted",
color="grey",
)
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def __call__(self):
"""Determine optimal mosaicity and domain size model (monochromatic)"""
RR = self.refinery.predict_for_reflection_table(self.reflections)
excursion_rad = RR["delpsical.rad"]
delta_psi_deg = excursion_rad * 180. / math.pi
print
print flex.max(delta_psi_deg), flex.min(delta_psi_deg)
mean_excursion = flex.mean(delta_psi_deg)
print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f" % (
mean_excursion, math.sqrt(flex.mean(RR["delpsical2"])))
crystal = self.experiments[0].crystal
beam = self.experiments[0].beam
miller_indices = self.reflections["miller_index"]
# FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection
two_thetas = crystal.get_unit_cell().two_theta(miller_indices,
beam.get_wavelength(),
deg=True)
dspacings = crystal.get_unit_cell().d(miller_indices)
dspace_sq = dspacings * dspacings
# First -- try to get a reasonable envelope for the observed excursions.
## minimum of three regions; maximum of 50 measurements in each bin
print "fitting parameters on %d spots" % len(excursion_rad)
n_bins = min(max(3, len(excursion_rad) // 25), 50)
bin_sz = len(excursion_rad) // n_bins
print "nbins", n_bins, "bin_sz", bin_sz
order = flex.sort_permutation(two_thetas)
two_thetas_env = flex.double()
dspacings_env = flex.double()
excursion_rads_env = flex.double()
for x in xrange(0, n_bins):
subset = order[x * bin_sz:(x + 1) * bin_sz]
two_thetas_env.append(flex.mean(two_thetas.select(subset)))
dspacings_env.append(flex.mean(dspacings.select(subset)))
excursion_rads_env.append(
flex.max(flex.abs(excursion_rad.select(subset))))
# Second -- parameter fit
## solve the normal equations
sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
sum_inv_u = flex.sum(dspacings_env)
sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
sum_te = flex.sum(excursion_rads_env)
Normal_Mat = sqr(
(sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
Vector = col((sum_te_u, sum_te))
solution = Normal_Mat.inverse() * Vector
s_ang = 1. / (2 * solution[0])
print "Best LSQ fit Scheerer domain size is %9.2f ang" % (s_ang)
tan_phi_rad = dspacings / (2. * s_ang)
tan_phi_deg = tan_phi_rad * 180. / math.pi
k_degrees = solution[1] * 180. / math.pi
print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f" % (
2 * k_degrees, k_degrees)
tan_outer_deg = tan_phi_deg + k_degrees
from xfel.mono_simulation.max_like import minimizer
# coerce the estimates to be positive for max-likelihood
lower_limit_domain_size = math.pow(
crystal.get_unit_cell().volume(),
1. / 3.) * 3 # params.refinement.domain_size_lower_limit
d_estimate = max(s_ang, lower_limit_domain_size)
M = minimizer(d_i=dspacings,
psi_i=excursion_rad,
eta_rad=abs(2. * solution[1]),
Deff=d_estimate)
print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % (
M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas))
tan_phi_rad_ML = dspacings / (2. / M.x[0])
tan_phi_deg_ML = tan_phi_rad_ML * 180. / math.pi
tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180. / math.pi
self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML
if self.graph_verbose: #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit
AD1TF7B_MAX2T = 30.
AD1TF7B_MAXDP = 1.
from matplotlib import pyplot as plt
plt.plot(two_thetas, delta_psi_deg, "bo")
minplot = flex.min(two_thetas)
plt.plot([0, minplot], [mean_excursion, mean_excursion], "k-")
LR = flex.linear_regression(two_thetas, delta_psi_deg)
model_y = LR.slope() * two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" %
(M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas)))
plt.plot(two_thetas, tan_phi_deg_ML, "r.")
plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
plt.plot(two_thetas, tan_outer_deg_ML, "g.")
plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
plt.xlim([0, AD1TF7B_MAX2T])
plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP])
plt.show()
plt.close()
from xfel.mono_simulation.util import green_curve_area
self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
print "The green curve area is ", self.green_curve_area
crystal._ML_half_mosaicity_deg = M.x[1] * 180. / (2. * math.pi)
crystal._ML_domain_size_ang = 2. / M.x[0]
self._ML_full_mosaicity_rad = M.x[1]
self._ML_domain_size_ang = 2. / M.x[0]
#params.refinement.mosaic.model_expansion_factor
"""The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes
as close as possible to # of observed reflections input, in the last integration call. Determine this by
inspecting the output log file interactively. Do not exceed the bare minimum threshold needed.
The intention is to find an optimal value, global for a given dataset."""
model_expansion_factor = 1.4
crystal._ML_half_mosaicity_deg *= model_expansion_factor
crystal._ML_domain_size_ang /= model_expansion_factor
return crystal
def plot_one_model(self, nrow, out):
fig = plt.subplot(self.gs[nrow * self.ncols])
two_thetas = self.reduction.get_two_theta_deg()
degrees = self.reduction.get_delta_psi_deg()
if self.color_encoding == "conventional":
positive = (self.reduction.i_sigi >= 0.)
fig.plot(two_thetas.select(positive), degrees.select(positive),
"bo")
fig.plot(two_thetas.select(~positive), degrees.select(~positive),
"r+")
elif self.color_encoding == "I/sigma":
positive = (self.reduction.i_sigi >= 0.)
tt_selected = two_thetas.select(positive)
dp_selected = degrees.select(positive)
i_sigi_select = self.reduction.i_sigi.select(positive)
order = flex.sort_permutation(i_sigi_select)
tt_selected = tt_selected.select(order)
dp_selected = dp_selected.select(order)
i_sigi_selected = i_sigi_select.select(order)
from matplotlib.colors import Normalize
dnorm = Normalize()
dcolors = i_sigi_selected.as_numpy_array()
dnorm.autoscale(dcolors)
N = len(dcolors)
CMAP = plt.get_cmap("rainbow")
if self.refined.get("partiality_array", None) is None:
for n in xrange(N):
fig.plot([tt_selected[n]], [dp_selected[n]],
color=CMAP(dnorm(dcolors[n])),
marker=".",
markersize=10)
else:
partials = self.refined.get("partiality_array")
partials_select = partials.select(positive)
partials_selected = partials_select.select(order)
assert len(partials) == len(positive)
for n in xrange(N):
fig.plot([tt_selected[n]], [dp_selected[n]],
color=CMAP(dnorm(dcolors[n])),
marker=".",
markersize=20 * partials_selected[n])
# change the markersize to indicate partiality.
negative = (self.reduction.i_sigi < 0.)
fig.plot(two_thetas.select(negative),
degrees.select(negative),
"r+",
linewidth=1)
else:
strong = (self.reduction.i_sigi >= 10.)
positive = ((~strong) & (self.reduction.i_sigi >= 0.))
negative = (self.reduction.i_sigi < 0.)
assert (strong.count(True) + positive.count(True) +
negative.count(True) == len(self.reduction.i_sigi))
fig.plot(two_thetas.select(positive), degrees.select(positive),
"bo")
fig.plot(two_thetas.select(strong),
degrees.select(strong),
marker='.',
linestyle='None',
markerfacecolor='#00ee00',
markersize=10)
fig.plot(two_thetas.select(negative), degrees.select(negative),
"r+")
# indicate the imposed resolution filter
wavelength = self.reduction.experiment.beam.get_wavelength()
imposed_res_filter = self.reduction.get_imposed_res_filter(out)
resolution_markers = [
a
for a in [imposed_res_filter,
self.reduction.measurements.d_min()] if a is not None
]
for RM in resolution_markers:
two_th = (180. / math.pi) * 2. * math.asin(wavelength / (2. * RM))
plt.plot([two_th, two_th],
[self.AD1TF7B_MAXDP * -0.8, self.AD1TF7B_MAXDP * 0.8],
'k-')
plt.text(two_th, self.AD1TF7B_MAXDP * -0.9, "%4.2f" % RM)
#indicate the linefit
mean = flex.mean(degrees)
minplot = flex.min(two_thetas)
plt.plot([0, minplot], [mean, mean], "k-")
LR = flex.linear_regression(two_thetas, degrees)
model_y = LR.slope() * two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
#Now let's take care of the red and green lines.
half_mosaic_rotation_deg = self.refined["half_mosaic_rotation_deg"]
mosaic_domain_size_ang = self.refined["mosaic_domain_size_ang"]
red_curve_domain_size_ang = self.refined.get(
"red_curve_domain_size_ang", mosaic_domain_size_ang)
a_step = self.AD1TF7B_MAX2T / 50.
a_range = flex.double([a_step * x for x in xrange(1, 50)
]) # domain two-theta array
#Bragg law [d=L/2sinTH]
d_spacing = (wavelength / (2. * flex.sin(math.pi * a_range / 360.)))
# convert two_theta to a delta-psi. Formula for Deffective [Dpsi=d/2Deff]
inner_phi_deg = flex.asin(
(d_spacing / (2. * red_curve_domain_size_ang))) * (180. / math.pi)
outer_phi_deg = flex.asin((d_spacing / (2.*mosaic_domain_size_ang)) + \
half_mosaic_rotation_deg*math.pi/180. )*(180./math.pi)
plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots\n%s" %
(2. * half_mosaic_rotation_deg, mosaic_domain_size_ang,
len(two_thetas), os.path.basename(self.reduction.filename)))
plt.plot(a_range, inner_phi_deg, "r-")
plt.plot(a_range, -inner_phi_deg, "r-")
plt.plot(a_range, outer_phi_deg, "g-")
plt.plot(a_range, -outer_phi_deg, "g-")
plt.xlim([0, self.AD1TF7B_MAX2T])
plt.ylim([-self.AD1TF7B_MAXDP, self.AD1TF7B_MAXDP])
#second plot shows histogram
fig = plt.subplot(self.gs[1 + nrow * self.ncols])
plt.xlim([-self.AD1TF7B_MAXDP, self.AD1TF7B_MAXDP])
nbins = 50
n, bins, patches = plt.hist(
dp_selected,
nbins,
range=(-self.AD1TF7B_MAXDP, self.AD1TF7B_MAXDP),
weights=self.reduction.i_sigi.select(positive),
normed=0,
facecolor="orange",
alpha=0.75)
#ersatz determine the median i_sigi point:
isi_positive = self.reduction.i_sigi.select(positive)
isi_order = flex.sort_permutation(isi_positive)
reordered = isi_positive.select(isi_order)
isi_median = reordered[int(len(isi_positive) * 0.9)]
isi_top_half_selection = (isi_positive > isi_median)
n, bins, patches = plt.hist(
dp_selected.select(isi_top_half_selection),
nbins,
range=(-self.AD1TF7B_MAXDP, self.AD1TF7B_MAXDP),
weights=isi_positive.select(isi_top_half_selection),
normed=0,
facecolor="#ff0000",
alpha=0.75)
plt.xlabel("(degrees)")
plt.title("Weighted histogram of Delta-psi")
def plot_one_model(self,nrow,out):
fig = plt.subplot(self.gs[nrow*self.ncols])
two_thetas = self.reduction.get_two_theta_deg()
degrees = self.reduction.get_delta_psi_deg()
if self.color_encoding=="conventional":
positive = (self.reduction.i_sigi>=0.)
fig.plot(two_thetas.select(positive), degrees.select(positive), "bo")
fig.plot(two_thetas.select(~positive), degrees.select(~positive), "r+")
elif self.color_encoding=="I/sigma":
positive = (self.reduction.i_sigi>=0.)
tt_selected = two_thetas.select(positive)
dp_selected = degrees.select(positive)
i_sigi_select = self.reduction.i_sigi.select(positive)
order = flex.sort_permutation(i_sigi_select)
tt_selected = tt_selected.select(order)
dp_selected = dp_selected.select(order)
i_sigi_selected = i_sigi_select.select(order)
from matplotlib.colors import Normalize
dnorm = Normalize()
dcolors = i_sigi_selected.as_numpy_array()
dnorm.autoscale(dcolors)
N = len(dcolors)
CMAP = plt.get_cmap("rainbow")
if self.refined.get("partiality_array",None) is None:
for n in xrange(N):
fig.plot([tt_selected[n]],[dp_selected[n]],
color=CMAP(dnorm(dcolors[n])),marker=".", markersize=10)
else:
partials = self.refined.get("partiality_array")
partials_select = partials.select(positive)
partials_selected = partials_select.select(order)
assert len(partials)==len(positive)
for n in xrange(N):
fig.plot([tt_selected[n]],[dp_selected[n]],
color=CMAP(dnorm(dcolors[n])),marker=".", markersize=20*partials_selected[n])
# change the markersize to indicate partiality.
negative = (self.reduction.i_sigi<0.)
fig.plot(two_thetas.select(negative), degrees.select(negative), "r+", linewidth=1)
else:
strong = (self.reduction.i_sigi>=10.)
positive = ((~strong) & (self.reduction.i_sigi>=0.))
negative = (self.reduction.i_sigi<0.)
assert (strong.count(True)+positive.count(True)+negative.count(True) ==
len(self.reduction.i_sigi))
fig.plot(two_thetas.select(positive), degrees.select(positive), "bo")
fig.plot(two_thetas.select(strong), degrees.select(strong), marker='.',linestyle='None',
markerfacecolor='#00ee00', markersize=10)
fig.plot(two_thetas.select(negative), degrees.select(negative), "r+")
# indicate the imposed resolution filter
wavelength = self.reduction.experiment.beam.get_wavelength()
imposed_res_filter = self.reduction.get_imposed_res_filter(out)
resolution_markers = [
a for a in [imposed_res_filter,self.reduction.measurements.d_min()] if a is not None]
for RM in resolution_markers:
two_th = (180./math.pi)*2.*math.asin(wavelength/(2.*RM))
plt.plot([two_th, two_th],[self.AD1TF7B_MAXDP*-0.8,self.AD1TF7B_MAXDP*0.8],'k-')
plt.text(two_th,self.AD1TF7B_MAXDP*-0.9,"%4.2f"%RM)
#indicate the linefit
mean = flex.mean(degrees)
minplot = flex.min(two_thetas)
plt.plot([0,minplot],[mean,mean],"k-")
LR = flex.linear_regression(two_thetas, degrees)
model_y = LR.slope()*two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
#Now let's take care of the red and green lines.
half_mosaic_rotation_deg = self.refined["half_mosaic_rotation_deg"]
mosaic_domain_size_ang = self.refined["mosaic_domain_size_ang"]
red_curve_domain_size_ang = self.refined.get("red_curve_domain_size_ang",mosaic_domain_size_ang)
a_step = self.AD1TF7B_MAX2T / 50.
a_range = flex.double([a_step*x for x in xrange(1,50)]) # domain two-theta array
#Bragg law [d=L/2sinTH]
d_spacing = (wavelength/(2.*flex.sin(math.pi*a_range/360.)))
# convert two_theta to a delta-psi. Formula for Deffective [Dpsi=d/2Deff]
inner_phi_deg = flex.asin((d_spacing / (2.*red_curve_domain_size_ang)) )*(180./math.pi)
outer_phi_deg = flex.asin((d_spacing / (2.*mosaic_domain_size_ang)) + \
half_mosaic_rotation_deg*math.pi/180. )*(180./math.pi)
plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots\n%s"%(
2.*half_mosaic_rotation_deg, mosaic_domain_size_ang, len(two_thetas),
os.path.basename(self.reduction.filename)))
plt.plot(a_range, inner_phi_deg, "r-")
plt.plot(a_range,-inner_phi_deg, "r-")
plt.plot(a_range, outer_phi_deg, "g-")
plt.plot(a_range, -outer_phi_deg, "g-")
plt.xlim([0,self.AD1TF7B_MAX2T])
plt.ylim([-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP])
#second plot shows histogram
fig = plt.subplot(self.gs[1+nrow*self.ncols])
plt.xlim([-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP])
nbins = 50
n,bins,patches = plt.hist(dp_selected, nbins,
range=(-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP),
weights=self.reduction.i_sigi.select(positive),
normed=0, facecolor="orange", alpha=0.75)
#ersatz determine the median i_sigi point:
isi_positive = self.reduction.i_sigi.select(positive)
isi_order = flex.sort_permutation(isi_positive)
reordered = isi_positive.select(isi_order)
isi_median = reordered[int(len(isi_positive)*0.9)]
isi_top_half_selection = (isi_positive>isi_median)
n,bins,patches = plt.hist(dp_selected.select(isi_top_half_selection), nbins,
range=(-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP),
weights=isi_positive.select(isi_top_half_selection),
normed=0, facecolor="#ff0000", alpha=0.75)
plt.xlabel("(degrees)")
plt.title("Weighted histogram of Delta-psi")
def run(args):
import libtbx.load_env
usage = "%s experiments.json indexed.pickle [options]" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0:
parser.print_help()
return
elif len(experiments) > 1:
raise Sorry("More than one experiment present")
experiment = experiments[0]
assert (len(reflections) == 1)
reflections = reflections[0]
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
if 'intensity.prf.value' in reflections:
intensities = reflections['intensity.prf.value']
variances = reflections['intensity.prf.variance']
sel = (variances > 0)
intensities = intensities.select(sel)
variances = variances.select(sel)
sigmas = flex.sqrt(variances)
indices = reflections['miller_index'].select(sel)
from cctbx import crystal, miller
crystal_symmetry = crystal.symmetry(
space_group=experiment.crystal.get_space_group(),
unit_cell=experiment.crystal.get_unit_cell())
miller_set = miller.set(crystal_symmetry=crystal_symmetry,
anomalous_flag=True,
indices=indices)
miller_array = miller.array(
miller_set=miller_set, data=intensities,
sigmas=sigmas).set_observation_type_xray_intensity()
#miller_array.setup_binner(n_bins=50, reflections_per_bin=100)
miller_array.setup_binner(auto_binning=True, n_bins=20)
result = miller_array.i_over_sig_i(use_binning=True)
result.show()
from cctbx import uctbx
d_star_sq_centre = result.binner.bin_centers(2)
i_over_sig_i = flex.double(
[d if d is not None else 0 for d in result.data[1:-1]])
sel = (i_over_sig_i > 0)
d_star_sq_centre = d_star_sq_centre.select(sel)
i_over_sig_i = i_over_sig_i.select(sel)
log_i_over_sig_i = flex.log(i_over_sig_i)
weights = result.binner.counts()[1:-1].as_double().select(sel)
fit = flex.linear_regression(d_star_sq_centre,
log_i_over_sig_i,
weights=weights)
m = fit.slope()
c = fit.y_intercept()
import math
y_cutoff = math.log(params.i_sigi_cutoff)
x_cutoff = (y_cutoff - c) / m
estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff)
print "estimated d_min: %.2f" % estimated_d_min
if params.plot:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot(list(d_star_sq_centre),
list(log_i_over_sig_i),
label=r"ln(I/sigI)")
ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()],
color='red')
ax.plot([x_cutoff, x_cutoff],
pyplot.ylim(),
color='grey',
linestyle='dashed')
ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff],
color='grey',
linestyle='dashed')
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(I/sigI)")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
pyplot.savefig("estimate_resolution_limit.png")
pyplot.clf()
def estimate_resolution_limit(reflections,
imageset,
ice_sel=None,
plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities / flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(d_star_sq.select(~ice_sel),
log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(d_star_sq,
target_n_per_bin=20,
max_slots=20,
min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1 - low_percentile_limit
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
#sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
#print "%.2f" %(sel.count(True)/sel_all.count(True))
if sel.count(True) == 0:
#outliers_all.set_selected(sel_all & ice_sel, True)
continue
#if i_slot > i_slot_max:
#break
#else:
#continue
outliers = wilson_outliers(reflections.select(sel_all),
ice_sel=ice_sel.select(sel_all))
#print "rejecting %d wilson outliers" %outliers.count(True)
outliers_all.set_selected(sel_all, outliers)
#if sel.count(True)/sel_all.count(True) < 0.25:
#outliers_all.set_selected(sel_all & ice_sel, True)
#from scitbx.math import median_statistics
#intensities_sel = intensities.select(sel)
#stats = median_statistics(intensities_sel)
#z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
#outliers = z_score > 3.5
#perm = flex.sort_permutation(intensities_sel)
##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
##print ' '.join('%.2f' %v for v in z_score.select(perm))
##print
isel = sel_all.iselection().select(~(outliers)
& ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
#fit_upper.show_summary()
#fit_lower.show_summary()
if m_upper == m_lower:
intersection = (-1, -1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
intersection = ((c_lower - c_upper) / (m_upper - m_lower),
(m_upper * c_lower - m_lower * c_upper) /
(m_upper - m_lower))
a = m_upper
c_ = c_upper
b = m_lower
d = c_lower
assert intersection == ((d - c_) / (a - b), (a * d - b * c_) / (a - b))
#inside = points_inside_envelope(
#d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper,
c_upper)
inside = inside & ~outliers_all
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
#d_star_sq_estimate = intersection[0]
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
#resolution_estimate = max(resolution_estimate, flex.min(d_spacings))
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
ax.scatter(d_star_sq.select(inside),
log_i_over_sigi.select(inside),
marker='+',
color='green')
ax.scatter(d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker='+',
color='black')
ax.scatter(d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker='+',
color='grey')
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')
if (intersection[0] <= ax.get_xlim()[1]
and intersection[1] <= ax.get_ylim()[1]):
ax.scatter([intersection[0]], [intersection[1]],
marker='x',
s=50,
color='b')
#ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
ax.set_xlabel('d_star_sq')
ax.set_ylabel('ln(I/sigI)')
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(uctbx.d_as_d_star_sq(slot.d_max),
0,
ax.get_ylim()[1],
linestyle='dotted',
color='grey')
ax.vlines(uctbx.d_as_d_star_sq(slot.d_min),
0,
ax.get_ylim()[1],
linestyle='dotted',
color='grey')
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def estimate_resolution_limit(reflections, imageset, ice_sel=None,
plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities/flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(
d_star_sq.select(~ice_sel), log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(
d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1-low_percentile_limit
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
#sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
#print "%.2f" %(sel.count(True)/sel_all.count(True))
if sel.count(True) == 0:
#outliers_all.set_selected(sel_all & ice_sel, True)
continue
#if i_slot > i_slot_max:
#break
#else:
#continue
outliers = wilson_outliers(
reflections.select(sel_all), ice_sel=ice_sel.select(sel_all))
#print "rejecting %d wilson outliers" %outliers.count(True)
outliers_all.set_selected(sel_all, outliers)
#if sel.count(True)/sel_all.count(True) < 0.25:
#outliers_all.set_selected(sel_all & ice_sel, True)
#from scitbx.math import median_statistics
#intensities_sel = intensities.select(sel)
#stats = median_statistics(intensities_sel)
#z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
#outliers = z_score > 3.5
#perm = flex.sort_permutation(intensities_sel)
##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
##print ' '.join('%.2f' %v for v in z_score.select(perm))
##print
isel = sel_all.iselection().select(~(outliers) & ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
#fit_upper.show_summary()
#fit_lower.show_summary()
if m_upper == m_lower:
intersection = (-1,-1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
intersection = (
(c_lower-c_upper)/(m_upper-m_lower),
(m_upper*c_lower-m_lower*c_upper)/(m_upper-m_lower))
a = m_upper
c_ = c_upper
b = m_lower
d = c_lower
assert intersection == ((d-c_)/(a-b), (a*d-b*c_)/(a-b))
#inside = points_inside_envelope(
#d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper, c_upper)
inside = inside & ~outliers_all
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
#d_star_sq_estimate = intersection[0]
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
#resolution_estimate = max(resolution_estimate, flex.min(d_spacings))
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
ax.scatter(d_star_sq.select(inside), log_i_over_sigi.select(inside),
marker='+', color='green')
ax.scatter(d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker='+', color='black')
ax.scatter(d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker='+', color='grey')
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')
if (intersection[0] <= ax.get_xlim()[1] and
intersection[1] <= ax.get_ylim()[1]):
ax.scatter([intersection[0]], [intersection[1]], marker='x', s=50, color='b')
#ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
ax.set_xlabel('d_star_sq')
ax.set_ylabel('ln(I/sigI)')
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(uctbx.d_as_d_star_sq(slot.d_max), 0, ax.get_ylim()[1],
linestyle='dotted', color='grey')
ax.vlines(uctbx.d_as_d_star_sq(slot.d_min), 0, ax.get_ylim()[1],
linestyle='dotted', color='grey')
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def __call__(self):
"""Determine optimal mosaicity and domain size model (monochromatic)"""
RR = self.refinery.predict_for_reflection_table(self.reflections)
excursion_rad = RR["delpsical.rad"]
delta_psi_deg = excursion_rad * 180./math.pi
print
print flex.max(delta_psi_deg), flex.min(delta_psi_deg)
mean_excursion = flex.mean(delta_psi_deg)
print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f"%(mean_excursion, math.sqrt(flex.mean(RR["delpsical2"])))
crystal = self.experiments[0].crystal
beam = self.experiments[0].beam
miller_indices = self.reflections["miller_index"]
# FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection
two_thetas = crystal.get_unit_cell().two_theta(miller_indices,beam.get_wavelength(),deg=True)
dspacings = crystal.get_unit_cell().d(miller_indices)
dspace_sq = dspacings * dspacings
# First -- try to get a reasonable envelope for the observed excursions.
## minimum of three regions; maximum of 50 measurements in each bin
print "fitting parameters on %d spots"%len(excursion_rad)
n_bins = min(max(3, len(excursion_rad)//25),50)
bin_sz = len(excursion_rad)//n_bins
print "nbins",n_bins,"bin_sz",bin_sz
order = flex.sort_permutation(two_thetas)
two_thetas_env = flex.double()
dspacings_env = flex.double()
excursion_rads_env = flex.double()
for x in xrange(0,n_bins):
subset = order[x*bin_sz:(x+1)*bin_sz]
two_thetas_env.append(flex.mean(two_thetas.select(subset)))
dspacings_env.append(flex.mean(dspacings.select(subset)))
excursion_rads_env.append(flex.max(flex.abs(excursion_rad.select(subset))))
# Second -- parameter fit
## solve the normal equations
sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
sum_inv_u = flex.sum(dspacings_env)
sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
sum_te = flex.sum(excursion_rads_env)
Normal_Mat = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
Vector = col((sum_te_u, sum_te))
solution = Normal_Mat.inverse() * Vector
s_ang = 1./(2*solution[0])
print "Best LSQ fit Scheerer domain size is %9.2f ang"%(
s_ang)
tan_phi_rad = dspacings / (2. * s_ang)
tan_phi_deg = tan_phi_rad * 180./math.pi
k_degrees = solution[1]* 180./math.pi
print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees)
tan_outer_deg = tan_phi_deg + k_degrees
from xfel.mono_simulation.max_like import minimizer
# coerce the estimates to be positive for max-likelihood
lower_limit_domain_size = math.pow(crystal.get_unit_cell().volume(),
1./3.)*3 # params.refinement.domain_size_lower_limit
d_estimate = max(s_ang, lower_limit_domain_size)
M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]),
Deff = d_estimate)
print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas))
tan_phi_rad_ML = dspacings / (2. / M.x[0])
tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi
tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi
self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML
if self.graph_verbose: #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit
AD1TF7B_MAX2T = 30.
AD1TF7B_MAXDP = 1.
from matplotlib import pyplot as plt
plt.plot(two_thetas, delta_psi_deg, "bo")
minplot = flex.min(two_thetas)
plt.plot([0,minplot],[mean_excursion,mean_excursion],"k-")
LR = flex.linear_regression(two_thetas, delta_psi_deg)
model_y = LR.slope()*two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)))
plt.plot(two_thetas, tan_phi_deg_ML, "r.")
plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
plt.plot(two_thetas, tan_outer_deg_ML, "g.")
plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
plt.xlim([0,AD1TF7B_MAX2T])
plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP])
plt.show()
plt.close()
from xfel.mono_simulation.util import green_curve_area
self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
print "The green curve area is ", self.green_curve_area
crystal._ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi)
crystal._ML_domain_size_ang = 2./M.x[0]
self._ML_full_mosaicity_rad = M.x[1]
self._ML_domain_size_ang = 2./M.x[0]
#params.refinement.mosaic.model_expansion_factor
"""The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes
as close as possible to # of observed reflections input, in the last integration call. Determine this by
inspecting the output log file interactively. Do not exceed the bare minimum threshold needed.
The intention is to find an optimal value, global for a given dataset."""
model_expansion_factor = 1.4
crystal._ML_half_mosaicity_deg *= model_expansion_factor
crystal._ML_domain_size_ang /= model_expansion_factor
return crystal
