def __call__(self, sigma_m):
"""Calculate the fraction of observed intensity for each observation.
Params:
sigma_m The mosaicity
Returns:
A list of log intensity fractions
"""
# Tiny value
TINY = 1e-10
assert sigma_m > TINY
# Calculate the two components to the fraction
a = scitbx.math.erf(self.e1 / sigma_m)
b = scitbx.math.erf(self.e2 / sigma_m)
# Calculate the fraction of observed reflection intensity
R = (a - b) / 2.0
# Set any points <= 0 to 1e-10 (otherwise will get a floating
# point error in log calculation below).
assert R.all_ge(0)
mask = R < TINY
assert mask.count(True) < len(mask)
R.set_selected(mask, TINY)
# Return the logarithm of r
return flex.log(R)
def target(self, log_sigma):
"""The target for minimization."""
sigma_m = math.exp(log_sigma[0])
# Tiny value
TINY = 1e-10
assert sigma_m > TINY
# Calculate the two components to the fraction
a = scitbx.math.erf(self.e1 / sigma_m)
b = scitbx.math.erf(self.e2 / sigma_m)
n = self.n
K = self.K
# Calculate the fraction of observed reflection intensity
zi = (a - b) / 2.0
# Set any points <= 0 to 1e-10 (otherwise will get a floating
# point error in log calculation below).
assert zi.all_ge(0)
mask = zi < TINY
assert mask.count(True) < len(mask)
zi.set_selected(mask, TINY)
# Compute the likelihood
#
# The likelihood here is a result of the sum of two log likelihood
# functions:
#
# The first is the same as the one in Kabsch2010 as applied to the
# reflection as a whole. This results in the term log(Z)
#
# The second is the likelihood for each reflection modelling as a Poisson
# distribtution with shape given by sigma M. This gives sum(ci log(zi)) -
# sum(ci)*log(sum(zi))
#
# If the reflection is recorded on 1 frame, the second component is zero
# and so the likelihood is dominated by the first term which can be seen
# as a prior for sigma, which accounts for which reflections were actually
# recorded.
#
L = 0
for j, (i0,
i1) in enumerate(zip(self.indices[:-1], self.indices[1:])):
selection = flex.size_t(range(i0, i1))
zj = zi.select(selection)
nj = n.select(selection)
kj = K[j]
Z = flex.sum(zj)
# L += flex.sum(nj * flex.log(zj)) - kj * Z
# L += flex.sum(nj * flex.log(zj)) - kj * math.log(Z)
L += flex.sum(
nj * flex.log(zj)) - kj * math.log(Z) + math.log(Z)
logger.debug("Sigma M: %f, log(L): %f", sigma_m * 180 / math.pi, L)
# Return the logarithm of r
return -L
def i_over_s_hist(self, rlist):
""" Analyse the correlations. """
I = rlist["intensity.sum.value"]
I_sig = flex.sqrt(rlist["intensity.sum.variance"])
I_over_S = I / I_sig
fig = pyplot.figure()
pyplot.title("Log I/Sigma histogram")
pyplot.hist(flex.log(I_over_S), bins=20)
pyplot.xlabel("Log I/Sigma")
pyplot.ylabel("# reflections")
fig.savefig(os.path.join(self.directory, "ioversigma_hist"))
pyplot.close()
def i_over_s_hist(self, rlist):
''' Analyse the correlations. '''
from os.path import join
I = rlist['intensity.sum.value']
I_sig = flex.sqrt(rlist['intensity.sum.variance'])
I_over_S = I / I_sig
fig = pyplot.figure()
pyplot.title("Log I/Sigma histogram")
pyplot.hist(flex.log(I_over_S), bins=20)
pyplot.xlabel("Log I/Sigma")
pyplot.ylabel("# reflections")
fig.savefig(join(self.directory, "ioversigma_hist"))
pyplot.close()
def plot_one_panel(self, ax, rlist):
I_sig = flex.sqrt(rlist['intensity.%s.variance' %intensity_type])
sel = I_sig > 0
rlist = rlist.select(sel)
I_sig = I_sig.select(sel)
I = rlist['intensity.%s.value' %intensity_type]
I_over_S = I / I_sig
x, y, z = rlist['xyzcal.px'].parts()
hex_ax = ax.hexbin(
x.as_numpy_array(), y.as_numpy_array(),
C=flex.log(I_over_S), gridsize=self.gridsize,
)
return hex_ax
def i_over_s_vs_z(self, rlist):
""" Plot I/Sigma vs Z. """
I = rlist["intensity.sum.value"]
I_sig = flex.sqrt(rlist["intensity.sum.variance"])
I_over_S = I / I_sig
x, y, z = rlist["xyzcal.px"].parts()
fig = pyplot.figure()
pyplot.title("Distribution of I/Sigma vs Z")
cax = pyplot.hexbin(z, flex.log(I_over_S), gridsize=100)
cax.axes.set_xlabel("z (images)")
cax.axes.set_ylabel("Log I/Sigma")
cbar = pyplot.colorbar(cax)
cbar.ax.set_ylabel("# reflections")
fig.savefig(os.path.join(self.directory, "ioversigma_vs_z.png"))
pyplot.close()
def _i_over_s_vs_xy_plot(ax, rlist, gridsize):
I_sig = flex.sqrt(rlist["intensity.%s.variance" % intensity_type])
sel = I_sig > 0
rlist = rlist.select(sel)
I_sig = I_sig.select(sel)
I = rlist["intensity.%s.value" % intensity_type]
I_over_S = I / I_sig
x, y, z = rlist["xyzcal.px"].parts()
hex_ax = ax.hexbin(
x.as_numpy_array(),
y.as_numpy_array(),
C=flex.log(I_over_S),
gridsize=gridsize,
)
return hex_ax
def i_over_s_vs_z(self, rlist):
''' Plot I/Sigma vs Z. '''
from os.path import join
I = rlist['intensity.sum.value']
I_sig = flex.sqrt(rlist['intensity.sum.variance'])
I_over_S = I / I_sig
x, y, z = rlist['xyzcal.px'].parts()
fig = pyplot.figure()
pyplot.title("Distribution of I/Sigma vs Z")
cax = pyplot.hexbin(z, flex.log(I_over_S), gridsize=100)
cax.axes.set_xlabel("z")
cax.axes.set_ylabel("Log I/Sigma")
cbar = pyplot.colorbar(cax)
cbar.ax.set_ylabel("# reflections")
fig.savefig(join(self.directory, "ioversigma_vs_z.png"))
pyplot.close()
def mean_vs_ios(self, rlist):
''' Analyse the correlations. '''
from os.path import join
MEAN = rlist['background.mean']
I = rlist['intensity.sum.value']
I_sig = flex.sqrt(rlist['intensity.sum.variance'])
I_over_S = I / I_sig
mask = I_over_S > 0.1
I_over_S = I_over_S.select(mask)
MEAN = MEAN.select(mask)
fig = pyplot.figure()
pyplot.title("Background Model mean vs Log I/Sigma")
cax = pyplot.hexbin(flex.log(I_over_S), MEAN, gridsize=100)
cbar = pyplot.colorbar(cax)
cax.axes.set_xlabel("Log I/Sigma")
cax.axes.set_ylabel("Background Model mean")
cbar.ax.set_ylabel("# reflections")
fig.savefig(join(self.directory, "background_model_mean_vs_ios.png"))
pyplot.close()
def mean_vs_ios(self, rlist):
""" Analyse the correlations. """
MEAN = rlist["background.mean"]
I = rlist["intensity.sum.value"]
I_sig = flex.sqrt(rlist["intensity.sum.variance"])
I_over_S = I / I_sig
mask = I_over_S > 0.1
I_over_S = I_over_S.select(mask)
MEAN = MEAN.select(mask)
fig = pyplot.figure()
pyplot.title("Background Model mean vs Log I/Sigma")
cax = pyplot.hexbin(flex.log(I_over_S), MEAN, gridsize=100)
cbar = pyplot.colorbar(cax)
cax.axes.set_xlabel("Log I/Sigma")
cax.axes.set_ylabel("Background Model mean")
cbar.ax.set_ylabel("# reflections")
fig.savefig(
os.path.join(self.directory, "background_model_mean_vs_ios.png"))
pyplot.close()
def ideal_reflection_corr_vs_ios(self, rlist, filename):
''' Analyse the correlations. '''
from os.path import join
if 'correlation.ideal.profile' in rlist:
corr = rlist['correlation.ideal.profile']
I = rlist['intensity.prf.value']
I_sig = flex.sqrt(rlist['intensity.prf.variance'])
mask = I_sig > 0
I = I.select(mask)
I_sig = I_sig.select(mask)
corr = corr.select(mask)
I_over_S = I / I_sig
mask = I_over_S > 0.1
I_over_S = I_over_S.select(mask)
corr = corr.select(mask)
pyplot.title("Reflection correlations vs Log I/Sigma")
cax = pyplot.hexbin(flex.log(I_over_S), corr, gridsize=100)
cbar = pyplot.colorbar(cax)
pyplot.xlabel("Log I/Sigma")
pyplot.ylabel("Correlation with reference profile")
cbar.ax.set_ylabel("# reflections")
pyplot.savefig(join(self.directory, "ideal_%s_corr_vs_ios.png" % filename))
pyplot.close()
def reflection_corr_vs_ios(self, rlist, filename):
""" Analyse the correlations. """
corr = rlist["profile.correlation"]
I = rlist["intensity.prf.value"]
I_sig = flex.sqrt(rlist["intensity.prf.variance"])
mask = I_sig > 0
I = I.select(mask)
I_sig = I_sig.select(mask)
corr = corr.select(mask)
I_over_S = I / I_sig
mask = I_over_S > 0.1
I_over_S = I_over_S.select(mask)
corr = corr.select(mask)
fig = pyplot.figure()
pyplot.title("Reflection correlations vs Log I/Sigma")
cax = pyplot.hexbin(flex.log(I_over_S), corr, gridsize=100)
cbar = pyplot.colorbar(cax)
cax.axes.set_xlabel("Log I/Sigma")
cax.axes.set_ylabel("Correlation with reference profile")
cbar.ax.set_ylabel("# reflections")
fig.savefig(
os.path.join(self.directory, "%s_corr_vs_ios.png" % filename))
pyplot.close()
def estimate_resolution_limit(reflections,
imageset,
ice_sel=None,
plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities / flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(d_star_sq.select(~ice_sel),
log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(d_star_sq,
target_n_per_bin=20,
max_slots=20,
min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1 - low_percentile_limit
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
#sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
#print "%.2f" %(sel.count(True)/sel_all.count(True))
if sel.count(True) == 0:
#outliers_all.set_selected(sel_all & ice_sel, True)
continue
#if i_slot > i_slot_max:
#break
#else:
#continue
outliers = wilson_outliers(reflections.select(sel_all),
ice_sel=ice_sel.select(sel_all))
#print "rejecting %d wilson outliers" %outliers.count(True)
outliers_all.set_selected(sel_all, outliers)
#if sel.count(True)/sel_all.count(True) < 0.25:
#outliers_all.set_selected(sel_all & ice_sel, True)
#from scitbx.math import median_statistics
#intensities_sel = intensities.select(sel)
#stats = median_statistics(intensities_sel)
#z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
#outliers = z_score > 3.5
#perm = flex.sort_permutation(intensities_sel)
##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
##print ' '.join('%.2f' %v for v in z_score.select(perm))
##print
isel = sel_all.iselection().select(~(outliers)
& ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
#fit_upper.show_summary()
#fit_lower.show_summary()
if m_upper == m_lower:
intersection = (-1, -1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
intersection = ((c_lower - c_upper) / (m_upper - m_lower),
(m_upper * c_lower - m_lower * c_upper) /
(m_upper - m_lower))
a = m_upper
c_ = c_upper
b = m_lower
d = c_lower
assert intersection == ((d - c_) / (a - b), (a * d - b * c_) / (a - b))
#inside = points_inside_envelope(
#d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper,
c_upper)
inside = inside & ~outliers_all
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
#d_star_sq_estimate = intersection[0]
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
#resolution_estimate = max(resolution_estimate, flex.min(d_spacings))
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
ax.scatter(d_star_sq.select(inside),
log_i_over_sigi.select(inside),
marker='+',
color='green')
ax.scatter(d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker='+',
color='black')
ax.scatter(d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker='+',
color='grey')
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')
if (intersection[0] <= ax.get_xlim()[1]
and intersection[1] <= ax.get_ylim()[1]):
ax.scatter([intersection[0]], [intersection[1]],
marker='x',
s=50,
color='b')
#ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
ax.set_xlabel('d_star_sq')
ax.set_ylabel('ln(I/sigI)')
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(uctbx.d_as_d_star_sq(slot.d_max),
0,
ax.get_ylim()[1],
linestyle='dotted',
color='grey')
ax.vlines(uctbx.d_as_d_star_sq(slot.d_min),
0,
ax.get_ylim()[1],
linestyle='dotted',
color='grey')
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def paper_test(B, S):
from numpy.random import poisson
from math import exp
background_shape = [1 for i in range(20)]
signal_shape = [1 if i >= 6 and i < 15 else 0 for i in range(20)]
background = [poisson(bb * B,1)[0] for bb in background_shape]
signal = [poisson(ss * S, 1)[0] for ss in signal_shape]
# background = [bb * B for bb in background_shape]
# signal = [ss * S for ss in signal_shape]
total = [b + s for b, s in zip(background, signal)]
# from matplotlib import pylab
# pylab.plot(total)
# pylab.plot(signal)
# pylab.plot(background)
# pylab.show()
total = [0, 1, 0, 0, 0, 0, 3, 1, 3, 3, 6, 6, 4, 1, 4, 0, 2, 0, 1, 1]
total = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]
# plot_prob_for_zero(total, background_shape, signal_shape)
# signal_shape = [exp(-(x - 10.0)**2 / (2*3.0**2)) for x in range(20)]
# signal_shape = [ss / sum(signal_shape) for ss in signal_shape]
# print signal_shape
#plot_valid(background_shape, signal_shape)
B = 155.0 / 296.0
from dials.array_family import flex
from math import log, factorial
V = flex.double(flex.grid(100, 100))
L = flex.double(flex.grid(100, 100))
DB = flex.double(flex.grid(100, 100))
DS = flex.double(flex.grid(100, 100))
P = flex.double(flex.grid(100, 100))
Fb = sum(background_shape)
Fs = sum(signal_shape)
SV = []
MASK = flex.bool(flex.grid(100, 100), False)
for BB in range(100):
for SS in range(100):
B = -5.0 + (BB) / 10.0
S = -5.0 + (SS) / 10.0
# SV.append(S)
VV = 0
LL = 0
DDB = 0
DDS = 0
for i in range(20):
s = signal_shape[i]
b = background_shape[i]
c = total[i]
if B*b + S*s <= 0:
# MASK[BB, SS] = True
LL = 0
if b == 0:
DDB += 0
else:
DDB += 1e7
if s == 0:
DDS += 0
else:
DDS += 1e7
# break
else:
# VV += (b + s)*c / (B*b + S*s)
# LL += c*log(B*b+S*s) - log(factorial(c)) - B*b - S*s
DDB += c*b/(B*b+S*s) - b
DDS += c*s/(B*b+S*s) - s
VV -= (Fb + Fs)
# print B, S, VV
# V[BB, SS] = abs(VV)
L[BB, SS] = LL
DB[BB,SS] = DDB
DS[BB,SS] = DDS
max_ind = flex.max_index(L)
j = max_ind // 100
i = max_ind % 100
print "Approx: ", (j+1) / 20.0, (i+1) / 20.0
print "Min/Max DB: ", flex.min(DB), flex.max(DB)
print "Min/Max DS: ", flex.min(DS), flex.max(DS)
from matplotlib import pylab
# pylab.imshow(flex.log(V).as_numpy_array(), extent=[0.05, 5.05, 5.05, 0.05])
# pylab.plot(SV, V)
# pylab.plot(SV, [0] * 100)
# pylab.show()
im = flex.exp(L).as_numpy_array()
import numpy
# im = numpy.ma.masked_array(im, mask=MASK.as_numpy_array())
# pylab.imshow(im)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
pylab.imshow(DB.as_numpy_array(), vmin=-100, vmax=100)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
pylab.contour(DB.as_numpy_array(), levels=[0], colors=['red'])
pylab.contour(DS.as_numpy_array(), levels=[0], colors=['black'])
pylab.show()
# im = numpy.ma.masked_array(DB.as_numpy_array(), mask=MASK.as_numpy_array())
# pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
# pylab.show()
# im = numpy.ma.masked_array(DS.as_numpy_array(), mask=MASK.as_numpy_array())
# pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
# pylab.show()
# exit(0)
S1, B1 = value(total, background_shape, signal_shape)
exit(0)
try:
S1, B1 = value(total, background_shape, signal_shape)
print "Result:"
print S1, B1
exit(0)
except Exception, e:
raise e
import sys
import traceback
traceback.print_exc()
from dials.array_family import flex
Fs = sum(signal_shape)
Fb = sum(background_shape)
Rs = flex.double(flex.grid(100, 100))
print "-----"
print B, S
# from matplotlib import pylab
# pylab.plot(total)
# pylab.show()
print background_shape
print signal_shape
print total
from math import exp, factorial, log
minx = -1
miny = -1
minr = 9999
for BB in range(0, 100):
for SS in range(0, 100):
B = -10 + (BB) / 5.0
S = -10 + (SS) / 5.0
L = 0
Fb2 = 0
Fs2 = 0
for i in range(len(total)):
c = total[i]
b = background_shape[i]
s = signal_shape[i]
# P = exp(-(B*b + S*s)) * (B*b+S*s)**c / factorial(c)
# print P
# if P > 0:
# L += log(P)
den = B*b + S*s
num1 = b*c
num2 = s*c
if den != 0:
Fb2 += num1 / den
Fs2 += num2 / den
R = (Fb2 - Fb)**2 + (Fs2 - Fs)**2
if R > 1000:
R = 0
# Rs[BB,SS] = L#R#Fs2 - Fs
Rs[BB,SS] = R#Fs2 - Fs
from matplotlib import pylab
pylab.imshow(flex.log(Rs).as_numpy_array(), extent=[-5,5,5,-5])
pylab.show()
exit(0)
def estimate(self, num_reflections):
"""
Estimate the model parameters
"""
from scitbx import simplex
from copy import deepcopy
from dials.array_family import flex
from random import uniform
# Select the reflections
reflections = self._select_reflections(self.reflections,
num_reflections)
# The number of parameters
num_parameters = len(self.history.names)
# Setup the starting simplex
if self.simplex is None:
self.simplex = []
for i in range(num_parameters + 1):
self.simplex.append(
flex.log(
flex.double([
uniform(0.0001, 0.01)
for j in range(num_parameters)
])))
class Evaluator(object):
"""
Evaluator to simplex
"""
def __init__(
self,
history,
experiment,
reflections,
num_integral,
use_mosaic_block_angular_spread,
use_wavelength_spread,
):
"""
Initialise
"""
from dials_scratch.jmp.profile_modelling import MLTarget3D
self.func = MLTarget3D(
experiment,
reflections,
num_integral=num_integral,
use_mosaic_block_angular_spread=
use_mosaic_block_angular_spread,
use_wavelength_spread=use_wavelength_spread,
)
self.count = 1
self.history = history
self.logL = None
def target(self, log_parameters):
"""
Compute the negative log likelihood
"""
from dials.array_family import flex
parameters = flex.exp(log_parameters)
self.logL = self.func.log_likelihood(parameters)
logL = flex.sum(self.logL)
self.count += 1
print(self.count, list(parameters), logL)
self.history.append(parameters, logL)
# Return negative log likelihood
return -logL
# Setup the simplex optimizer
optimizer = simplex.simplex_opt(
num_parameters,
matrix=self.simplex,
evaluator=Evaluator(
self.history,
self.experiments[0],
reflections,
num_integral=self.num_integral,
use_mosaic_block_angular_spread=self.
use_mosaic_block_angular_spread,
use_wavelength_spread=self.use_wavelength_spread,
),
tolerance=1e-7,
)
# Get the solution
self.parameters = flex.exp(optimizer.get_solution())
# Get the final simplex
self.simplex = optimizer.matrix
# Save the likelihood for each reflection
self.log_likelihood = optimizer.evaluator.logL
def run(args):
import libtbx.load_env
usage = "%s experiments.json indexed.pickle [options]" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0:
parser.print_help()
return
elif len(experiments) > 1:
raise Sorry("More than one experiment present")
experiment = experiments[0]
assert (len(reflections) == 1)
reflections = reflections[0]
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
if 'intensity.prf.value' in reflections:
intensities = reflections['intensity.prf.value']
variances = reflections['intensity.prf.variance']
sel = (variances > 0)
intensities = intensities.select(sel)
variances = variances.select(sel)
sigmas = flex.sqrt(variances)
indices = reflections['miller_index'].select(sel)
from cctbx import crystal, miller
crystal_symmetry = crystal.symmetry(
space_group=experiment.crystal.get_space_group(),
unit_cell=experiment.crystal.get_unit_cell())
miller_set = miller.set(crystal_symmetry=crystal_symmetry,
anomalous_flag=True,
indices=indices)
miller_array = miller.array(
miller_set=miller_set, data=intensities,
sigmas=sigmas).set_observation_type_xray_intensity()
#miller_array.setup_binner(n_bins=50, reflections_per_bin=100)
miller_array.setup_binner(auto_binning=True, n_bins=20)
result = miller_array.i_over_sig_i(use_binning=True)
result.show()
from cctbx import uctbx
d_star_sq_centre = result.binner.bin_centers(2)
i_over_sig_i = flex.double(
[d if d is not None else 0 for d in result.data[1:-1]])
sel = (i_over_sig_i > 0)
d_star_sq_centre = d_star_sq_centre.select(sel)
i_over_sig_i = i_over_sig_i.select(sel)
log_i_over_sig_i = flex.log(i_over_sig_i)
weights = result.binner.counts()[1:-1].as_double().select(sel)
fit = flex.linear_regression(d_star_sq_centre,
log_i_over_sig_i,
weights=weights)
m = fit.slope()
c = fit.y_intercept()
import math
y_cutoff = math.log(params.i_sigi_cutoff)
x_cutoff = (y_cutoff - c) / m
estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff)
print "estimated d_min: %.2f" % estimated_d_min
if params.plot:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot(list(d_star_sq_centre),
list(log_i_over_sig_i),
label=r"ln(I/sigI)")
ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()],
color='red')
ax.plot([x_cutoff, x_cutoff],
pyplot.ylim(),
color='grey',
linestyle='dashed')
ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff],
color='grey',
linestyle='dashed')
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(I/sigI)")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
pyplot.savefig("estimate_resolution_limit.png")
pyplot.clf()
def plot_spotfinder_stats(
stats,
spot_length_stats,
run_tags=[],
run_statuses=[],
minimalist=False,
interactive=True,
xsize=30,
ysize=10,
high_vis=False,
title=None,
ext='cbf',
):
plot_ratio = max(min(xsize, ysize) / 2.5, 3)
if high_vis:
spot_ratio = plot_ratio * 4
text_ratio = plot_ratio * 4.5
else:
spot_ratio = plot_ratio * 2
text_ratio = plot_ratio * 3
t, n_strong, enough_rate, n_min, window, lengths, boundaries, run_numbers = stats
t_lengths, spot_lengths, spot_intensities = spot_length_stats
# set up coloring of spot lengths by intensities
from matplotlib.cm import ScalarMappable
mappable = ScalarMappable(cmap="plasma")
cmap = mappable.to_rgba(-flex.log(spot_intensities))
if len(t) == 0:
return None
n_runs = len(boundaries) // 2
if len(run_tags) != n_runs:
run_tags = [[] for i in range(n_runs)]
if len(run_statuses) != n_runs:
run_statuses = [None for i in range(n_runs)]
if minimalist:
print "Minimalist mode activated."
f, (ax1, ax2) = plt.subplots(2, sharex=True, sharey=False)
axset = (ax1, ax2)
else:
f, (ax1, ax2, ax3) = plt.subplots(3, sharex=True, sharey=False)
axset = (ax1, ax2, ax3)
for a in axset:
a.tick_params(axis='x', which='both', bottom='off', top='off')
ax1.scatter(t, n_strong, edgecolors="none", color='deeppink', s=spot_ratio)
ax1.set_ylim(ymin=0)
ax1.axis('tight')
ax1.set_ylabel("# strong spots", fontsize=text_ratio)
ax1_twin = ax1.twinx()
ax1_twin.plot(t, enough_rate * 100, color='orange')
ax1_twin.set_ylim(ymin=0)
ax1_twin.set_ylabel("%% images with\n>=%d spots" % n_min,
fontsize=text_ratio)
if spot_length_stats:
ax2.scatter(t_lengths,
spot_lengths,
edgecolors="none",
color=cmap,
s=spot_ratio)
ax2.set_ylim(ymin=0)
ax2.axis('tight')
ax2.set_ylabel("spot lengths\n(pixels)", fontsize=text_ratio)
f.subplots_adjust(hspace=0)
# add lines and text summaries at the timestamp boundaries
if not minimalist:
for boundary in boundaries:
if boundary is not None:
for a in (ax1, ax2):
a.axvline(x=boundary,
ymin=0,
ymax=2,
linewidth=1,
color='k')
run_starts = boundaries[0::2]
run_ends = boundaries[1::2]
start = 0
end = -1
for idx in range(len(run_numbers)):
start_t = run_starts[idx]
end_t = run_ends[idx]
if start_t is None or end_t is None: continue
end += lengths[idx]
slice_t = t[start:end + 1]
slice_enough = n_strong[start:end + 1] > n_min
tags = run_tags[idx]
status = run_statuses[idx]
if status == "DONE":
status_color = 'blue'
elif status in ["RUN", "PEND", "SUBMITTED"]:
status_color = 'green'
elif status is None:
status_color = 'black'
else:
status_color = 'red'
if minimalist:
ax2.set_xlabel(
"timestamp (s)\n# images shown as all (%3.1f Angstroms)" %
d_min,
fontsize=text_ratio)
ax2.set_yticks([])
else:
ax3.text(start_t,
2.85,
" " + ", ".join(tags) + " [%s]" % status,
fontsize=text_ratio,
color=status_color)
ax3.text(start_t,
.85,
"run %d" % run_numbers[idx],
fontsize=text_ratio)
ax3.text(start_t,
.65,
"%d images" % lengths[idx],
fontsize=text_ratio)
ax3.text(start_t,
.45,
"%d n>=%d" % (slice_enough.count(True), n_min),
fontsize=text_ratio)
if spot_length_stats and len(spot_lengths) > 0:
ax3.text(start_t,
.25,
"avg %d px" %
int(flex.sum(spot_lengths) / len(spot_lengths)),
fontsize=text_ratio)
ax3.set_xlabel("timestamp (s)", fontsize=text_ratio)
ax3.set_yticks([])
for item in axset:
item.tick_params(labelsize=text_ratio)
start += lengths[idx]
if title is not None:
plt.title(title)
f.set_size_inches(xsize, ysize)
f.savefig("spotfinder_tmp.png", bbox_inches='tight', dpi=100)
plt.close(f)
return "spotfinder_tmp.png"
def estimate_resolution_limit(reflections, ice_sel=None, plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections["intensity.sum.value"]
variances = reflections["intensity.sum.variance"]
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities / flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(d_star_sq.select(~ice_sel),
log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(d_star_sq,
target_n_per_bin=20,
max_slots=20,
min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1 - low_percentile_limit
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
if sel.count(True) == 0:
continue
outliers = wilson_outliers(reflections.select(sel_all),
ice_sel=ice_sel.select(sel_all))
outliers_all.set_selected(sel_all, outliers)
isel = sel_all.iselection().select(~(outliers)
& ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
if m_upper == m_lower:
intersection = (-1, -1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
# with:
# a_ = m_upper
# b_ = m_lower
# c_ = c_upper
# d_ = c_lower
# intersection == ((d_ - c_) / (a_ - b_), (a_ * d_ - b_ * c_) / (a_ - b_))
intersection = (
(c_lower - c_upper) / (m_upper - m_lower),
(m_upper * c_lower - m_lower * c_upper) / (m_upper - m_lower),
)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper,
c_upper)
inside = inside & ~outliers_all & ~ice_sel
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
if plot_filename is not None:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(d_star_sq, log_i_over_sigi, marker="+")
ax.scatter(
d_star_sq.select(inside),
log_i_over_sigi.select(inside),
marker="+",
color="green",
)
ax.scatter(
d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker="+",
color="black",
)
ax.scatter(
d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker="+",
color="grey",
)
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker="+", color="red")
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker="+", color="red")
if intersection[0] <= ax.get_xlim(
)[1] and intersection[1] <= ax.get_ylim()[1]:
ax.scatter([intersection[0]], [intersection[1]],
marker="x",
s=50,
color="b")
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color="red")
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color="red")
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(I/sigI)")
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(
uctbx.d_as_d_star_sq(slot.d_max),
0,
ax.get_ylim()[1],
linestyle="dotted",
color="grey",
)
ax.vlines(
uctbx.d_as_d_star_sq(slot.d_min),
0,
ax.get_ylim()[1],
linestyle="dotted",
color="grey",
)
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def estimate_resolution_limit(reflections, imageset, ice_sel=None,
plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities/flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(
d_star_sq.select(~ice_sel), log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(
d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1-low_percentile_limit
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
#sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
#print "%.2f" %(sel.count(True)/sel_all.count(True))
if sel.count(True) == 0:
#outliers_all.set_selected(sel_all & ice_sel, True)
continue
#if i_slot > i_slot_max:
#break
#else:
#continue
outliers = wilson_outliers(
reflections.select(sel_all), ice_sel=ice_sel.select(sel_all))
#print "rejecting %d wilson outliers" %outliers.count(True)
outliers_all.set_selected(sel_all, outliers)
#if sel.count(True)/sel_all.count(True) < 0.25:
#outliers_all.set_selected(sel_all & ice_sel, True)
#from scitbx.math import median_statistics
#intensities_sel = intensities.select(sel)
#stats = median_statistics(intensities_sel)
#z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
#outliers = z_score > 3.5
#perm = flex.sort_permutation(intensities_sel)
##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
##print ' '.join('%.2f' %v for v in z_score.select(perm))
##print
isel = sel_all.iselection().select(~(outliers) & ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
#fit_upper.show_summary()
#fit_lower.show_summary()
if m_upper == m_lower:
intersection = (-1,-1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
intersection = (
(c_lower-c_upper)/(m_upper-m_lower),
(m_upper*c_lower-m_lower*c_upper)/(m_upper-m_lower))
a = m_upper
c_ = c_upper
b = m_lower
d = c_lower
assert intersection == ((d-c_)/(a-b), (a*d-b*c_)/(a-b))
#inside = points_inside_envelope(
#d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper, c_upper)
inside = inside & ~outliers_all
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
#d_star_sq_estimate = intersection[0]
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
#resolution_estimate = max(resolution_estimate, flex.min(d_spacings))
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
ax.scatter(d_star_sq.select(inside), log_i_over_sigi.select(inside),
marker='+', color='green')
ax.scatter(d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker='+', color='black')
ax.scatter(d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker='+', color='grey')
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')
if (intersection[0] <= ax.get_xlim()[1] and
intersection[1] <= ax.get_ylim()[1]):
ax.scatter([intersection[0]], [intersection[1]], marker='x', s=50, color='b')
#ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
ax.set_xlabel('d_star_sq')
ax.set_ylabel('ln(I/sigI)')
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(uctbx.d_as_d_star_sq(slot.d_max), 0, ax.get_ylim()[1],
linestyle='dotted', color='grey')
ax.vlines(uctbx.d_as_d_star_sq(slot.d_min), 0, ax.get_ylim()[1],
linestyle='dotted', color='grey')
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def run(args):
import libtbx.load_env
usage = "%s experiments.json indexed.pickle [options]" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0:
parser.print_help()
return
elif len(experiments) > 1:
raise Sorry("More than one experiment present")
experiment = experiments[0]
assert(len(reflections) == 1)
reflections = reflections[0]
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
if 'intensity.prf.value' in reflections:
intensities = reflections['intensity.prf.value']
variances = reflections['intensity.prf.variance']
sel = (variances > 0)
intensities = intensities.select(sel)
variances = variances.select(sel)
sigmas = flex.sqrt(variances)
indices = reflections['miller_index'].select(sel)
from cctbx import crystal, miller
crystal_symmetry = crystal.symmetry(
space_group=experiment.crystal.get_space_group(),
unit_cell=experiment.crystal.get_unit_cell())
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
anomalous_flag=True,
indices=indices)
miller_array = miller.array(
miller_set=miller_set,
data=intensities,
sigmas=sigmas).set_observation_type_xray_intensity()
#miller_array.setup_binner(n_bins=50, reflections_per_bin=100)
miller_array.setup_binner(auto_binning=True, n_bins=20)
result = miller_array.i_over_sig_i(use_binning=True)
result.show()
from cctbx import uctbx
d_star_sq_centre = result.binner.bin_centers(2)
i_over_sig_i = flex.double(
[d if d is not None else 0 for d in result.data[1:-1]])
sel = (i_over_sig_i > 0)
d_star_sq_centre = d_star_sq_centre.select(sel)
i_over_sig_i = i_over_sig_i.select(sel)
log_i_over_sig_i = flex.log(i_over_sig_i)
weights = result.binner.counts()[1:-1].as_double().select(sel)
fit = flex.linear_regression(
d_star_sq_centre, log_i_over_sig_i, weights=weights)
m = fit.slope()
c = fit.y_intercept()
import math
y_cutoff = math.log(params.i_sigi_cutoff)
x_cutoff = (y_cutoff - c)/m
estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff)
print "estimated d_min: %.2f" %estimated_d_min
if params.plot:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.plot(
list(d_star_sq_centre),
list(log_i_over_sig_i),
label=r"ln(I/sigI)")
ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()], color='red')
ax.plot([x_cutoff, x_cutoff], pyplot.ylim(), color='grey', linestyle='dashed')
ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff], color='grey', linestyle='dashed')
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(I/sigI)")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
pyplot.savefig("estimate_resolution_limit.png")
pyplot.clf()
def paper_test(B, S):
from numpy.random import poisson
from math import exp
background_shape = [1 for i in range(20)]
signal_shape = [1 if i >= 6 and i < 15 else 0 for i in range(20)]
background = [poisson(bb * B, 1)[0] for bb in background_shape]
signal = [poisson(ss * S, 1)[0] for ss in signal_shape]
# background = [bb * B for bb in background_shape]
# signal = [ss * S for ss in signal_shape]
total = [b + s for b, s in zip(background, signal)]
# from matplotlib import pylab
# pylab.plot(total)
# pylab.plot(signal)
# pylab.plot(background)
# pylab.show()
total = [0, 1, 0, 0, 0, 0, 3, 1, 3, 3, 6, 6, 4, 1, 4, 0, 2, 0, 1, 1]
total = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]
# plot_prob_for_zero(total, background_shape, signal_shape)
# signal_shape = [exp(-(x - 10.0)**2 / (2*3.0**2)) for x in range(20)]
# signal_shape = [ss / sum(signal_shape) for ss in signal_shape]
# print signal_shape
# plot_valid(background_shape, signal_shape)
B = 155.0 / 296.0
from dials.array_family import flex
from math import log, factorial
V = flex.double(flex.grid(100, 100))
L = flex.double(flex.grid(100, 100))
DB = flex.double(flex.grid(100, 100))
DS = flex.double(flex.grid(100, 100))
P = flex.double(flex.grid(100, 100))
Fb = sum(background_shape)
Fs = sum(signal_shape)
SV = []
MASK = flex.bool(flex.grid(100, 100), False)
for BB in range(100):
for SS in range(100):
B = -5.0 + (BB) / 10.0
S = -5.0 + (SS) / 10.0
# SV.append(S)
VV = 0
LL = 0
DDB = 0
DDS = 0
for i in range(20):
s = signal_shape[i]
b = background_shape[i]
c = total[i]
if B * b + S * s <= 0:
# MASK[BB, SS] = True
LL = 0
if b == 0:
DDB += 0
else:
DDB += 1e7
if s == 0:
DDS += 0
else:
DDS += 1e7
# break
else:
# VV += (b + s)*c / (B*b + S*s)
# LL += c*log(B*b+S*s) - log(factorial(c)) - B*b - S*s
DDB += c * b / (B * b + S * s) - b
DDS += c * s / (B * b + S * s) - s
VV -= Fb + Fs
# print B, S, VV
# V[BB, SS] = abs(VV)
L[BB, SS] = LL
DB[BB, SS] = DDB
DS[BB, SS] = DDS
max_ind = flex.max_index(L)
j = max_ind // 100
i = max_ind % 100
print("Approx: ", (j + 1) / 20.0, (i + 1) / 20.0)
print("Min/Max DB: ", flex.min(DB), flex.max(DB))
print("Min/Max DS: ", flex.min(DS), flex.max(DS))
from matplotlib import pylab
# pylab.imshow(flex.log(V).as_numpy_array(), extent=[0.05, 5.05, 5.05, 0.05])
# pylab.plot(SV, V)
# pylab.plot(SV, [0] * 100)
# pylab.show()
im = flex.exp(L).as_numpy_array()
import numpy
# im = numpy.ma.masked_array(im, mask=MASK.as_numpy_array())
# pylab.imshow(im)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
pylab.imshow(
DB.as_numpy_array(), vmin=-100, vmax=100
) # , extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
pylab.contour(DB.as_numpy_array(), levels=[0], colors=["red"])
pylab.contour(DS.as_numpy_array(), levels=[0], colors=["black"])
pylab.show()
# im = numpy.ma.masked_array(DB.as_numpy_array(), mask=MASK.as_numpy_array())
# pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
# pylab.show()
# im = numpy.ma.masked_array(DS.as_numpy_array(), mask=MASK.as_numpy_array())
# pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
# pylab.show()
# exit(0)
S1, B1 = value(total, background_shape, signal_shape)
exit(0)
try:
S1, B1 = value(total, background_shape, signal_shape)
print("Result:")
print(S1, B1)
exit(0)
except Exception as e:
raise e
import sys
import traceback
traceback.print_exc()
from dials.array_family import flex
Fs = sum(signal_shape)
Fb = sum(background_shape)
Rs = flex.double(flex.grid(100, 100))
print("-----")
print(B, S)
# from matplotlib import pylab
# pylab.plot(total)
# pylab.show()
print(background_shape)
print(signal_shape)
print(total)
from math import exp, factorial, log
minx = -1
miny = -1
minr = 9999
for BB in range(0, 100):
for SS in range(0, 100):
B = -10 + (BB) / 5.0
S = -10 + (SS) / 5.0
L = 0
Fb2 = 0
Fs2 = 0
for i in range(len(total)):
c = total[i]
b = background_shape[i]
s = signal_shape[i]
# P = exp(-(B*b + S*s)) * (B*b+S*s)**c / factorial(c)
# print P
# if P > 0:
# L += log(P)
den = B * b + S * s
num1 = b * c
num2 = s * c
if den != 0:
Fb2 += num1 / den
Fs2 += num2 / den
R = (Fb2 - Fb) ** 2 + (Fs2 - Fs) ** 2
if R > 1000:
R = 0
# Rs[BB,SS] = L#R#Fs2 - Fs
Rs[BB, SS] = R # Fs2 - Fs
from matplotlib import pylab
pylab.imshow(flex.log(Rs).as_numpy_array(), extent=[-5, 5, 5, -5])
pylab.show()
exit(0)
exit(0)
# print S, B, sum(signal), sum(background), S1, B1
return S, B, sum(signal), sum(background), S1, B1
