def target(self, vector):
""" Compute the functional by first applying the current values for the sd parameters
to the input data, then computing the complete set of normalized deviations and finally
using those normalized deviations to compute the functional."""
values = self.parameterization(vector)
if values.SDFAC < 0 or values.SDB < 0 or values.SDADD < 0:
f = 1e6
else:
data = self.apply_sd_error_params(self.data, values)
all_sigmas_normalized = compute_normalized_deviations(
data, self.indices)
f = 0
for bin in self.intensity_bin_selections:
binned_normalized_sigmas = all_sigmas_normalized.select(bin)
n = len(binned_normalized_sigmas)
if n == 0: continue
# weighting scheme from Evans, 2011
w = math.sqrt(n)
# functional is weight * (1-rms(normalized_sigmas))^s summed over all intensitiy bins
f += w * ((1 - math.sqrt(
flex.mean(binned_normalized_sigmas *
binned_normalized_sigmas)))**2)
print >> self.log, "f: % 12.1f," % f,
values.show(self.log)
return f
def adjust_errors(self):
"""
Adjust sigmas according to Evans, 2011 Acta D and Evans and Murshudov, 2013 Acta D
"""
print >> self.log, "Starting adjust_errors"
print >> self.log, "Computing initial estimates of sdfac, sdb and sdadd"
values = self.parameterization(flex.double(self.get_initial_sdparams_estimates()))
print >> self.log, "Initial estimates:",
values.show(self.log)
print >> self.log, "Refining error correction parameters sdfac, sdb, and sdadd"
sels, binned_intensities = self.get_binned_intensities()
seed = self.scaler.params.raw_data.error_models.sdfac_refine.random_seed
minimizer = self.run_minimzer(values, sels, seed=seed)
values = minimizer.get_refined_params()
print >> self.log, "Final",
values.show(self.log)
print >> self.log, "Applying sdfac/sdb/sdadd 1"
self.scaler.ISIGI = minimizer.apply_sd_error_params(self.scaler.ISIGI, values)
self.scaler.summed_weight= flex.double(self.scaler.n_refl, 0.)
self.scaler.summed_wt_I = flex.double(self.scaler.n_refl, 0.)
print >> self.log, "Applying sdfac/sdb/sdadd 2"
for hkl_id in xrange(self.scaler.n_refl):
hkl = self.scaler.miller_set.indices()[hkl_id]
if hkl not in self.scaler.ISIGI: continue
n = len(self.scaler.ISIGI[hkl])
for i in xrange(n):
Intensity = self.scaler.ISIGI[hkl][i][0] # scaled intensity
sigma = Intensity / self.scaler.ISIGI[hkl][i][1] # corrected sigma
variance = sigma * sigma
self.scaler.summed_wt_I[hkl_id] += Intensity / variance
self.scaler.summed_weight[hkl_id] += 1 / variance
if False:
# validate using http://ccp4wiki.org/~ccp4wiki/wiki/index.php?title=Symmetry%2C_Scale%2C_Merge#Analysis_of_Standard_Deviations
print >> self.log, "Validating"
from matplotlib import pyplot as plt
all_sigmas_normalized = compute_normalized_deviations(self.scaler.ISIGI, self.scaler.miller_set.indices())
plt.hist(all_sigmas_normalized, bins=100)
plt.figure()
binned_rms_normalized_sigmas = []
for i, sel in enumerate(sels):
binned_rms_normalized_sigmas.append(math.sqrt(flex.mean(all_sigmas_normalized.select(sel)*all_sigmas_normalized.select(sel))))
plt.plot(binned_intensities, binned_rms_normalized_sigmas, 'o')
plt.show()
all_sigmas_normalized = all_sigmas_normalized.select(all_sigmas_normalized != 0)
self.normal_probability_plot(all_sigmas_normalized, (-0.5, 0.5), plot = True)
def get_initial_sdparams_estimates(self):
"""
Use normal probability analysis to compute intial sdfac and sdadd parameters.
"""
from xfel import compute_normalized_deviations
all_sigmas_normalized = compute_normalized_deviations(self.scaler.ISIGI, self.scaler.miller_set.indices())
assert ((all_sigmas_normalized > 0) | (all_sigmas_normalized <= 0)).count(True) == len(all_sigmas_normalized) # no nans allowed
# remove zeros (miller indices with only one observation will have a normalized deviation of 0 which shouldn't contribute to
# the normal probability plot analysis and initial parameter estimation
all_sigmas_normalized = all_sigmas_normalized.select(all_sigmas_normalized != 0)
corr, slope, offset = self.normal_probability_plot(all_sigmas_normalized, (-0.5, 0.5))
sdfac = 1/slope
sdadd = offset
#sdadd = -offset/slope
sdb = math.sqrt(sdadd)
return sdfac, sdb, sdadd
def adjust_errors(self):
"""
Adjust sigmas according to Evans, 2011 Acta D and Evans and Murshudov, 2013 Acta D
"""
print >> self.log, "Starting adjust_errors"
print >> self.log, "Computing initial estimates of sdfac, sdb and sdadd"
values = self.parameterization(
flex.double(self.get_initial_sdparams_estimates()))
print >> self.log, "Initial estimates:",
values.show(self.log)
print >> self.log, "Refining error correction parameters sdfac, sdb, and sdadd"
sels, binned_intensities = self.get_binned_intensities()
seed = self.scaler.params.raw_data.error_models.sdfac_refine.random_seed
minimizer = self.run_minimzer(values, sels, seed=seed)
values = minimizer.get_refined_params()
print >> self.log, "Final",
values.show(self.log)
print >> self.log, "Applying sdfac/sdb/sdadd 1"
minimizer.apply_sd_error_params(self.scaler.ISIGI, values)
self.scaler.summed_weight = flex.double(self.scaler.n_refl, 0.)
self.scaler.summed_wt_I = flex.double(self.scaler.n_refl, 0.)
print >> self.log, "Applying sdfac/sdb/sdadd 2"
for i in range(len(self.scaler.ISIGI)):
hkl_id = self.scaler.ISIGI['miller_id'][i]
Intensity = self.scaler.ISIGI['scaled_intensity'][
i] # scaled intensity
sigma = Intensity / self.scaler.ISIGI['isigi'][
i] # corrected sigma
variance = sigma * sigma
self.scaler.summed_wt_I[hkl_id] += Intensity / variance
self.scaler.summed_weight[hkl_id] += 1 / variance
if self.scaler.params.raw_data.error_models.sdfac_refine.plot_refinement_steps:
from matplotlib.pyplot import cm
from matplotlib import pyplot as plt
import numpy as np
for i in range(2):
f = plt.figure(i)
lines = plt.gca().get_lines()
color = cm.rainbow(np.linspace(0, 1, len(lines)))
for line, c in zip(reversed(lines), color):
line.set_color(c)
plt.ioff()
plt.show()
if False:
# validate using http://ccp4wiki.org/~ccp4wiki/wiki/index.php?title=Symmetry%2C_Scale%2C_Merge#Analysis_of_Standard_Deviations
print >> self.log, "Validating"
from matplotlib import pyplot as plt
all_sigmas_normalized = compute_normalized_deviations(
self.scaler.ISIGI, self.scaler.miller_set.indices())
plt.hist(all_sigmas_normalized, bins=100)
plt.figure()
binned_rms_normalized_sigmas = []
for i, sel in enumerate(sels):
binned_rms_normalized_sigmas.append(
math.sqrt(
flex.mean(
all_sigmas_normalized.select(sel) *
all_sigmas_normalized.select(sel))))
plt.plot(binned_intensities, binned_rms_normalized_sigmas, 'o')
plt.show()
all_sigmas_normalized = all_sigmas_normalized.select(
all_sigmas_normalized != 0)
self.normal_probability_plot(all_sigmas_normalized, (-0.5, 0.5),
plot=True)
def compute_normalized_deviations(self, data, indices): from xfel import compute_normalized_deviations return compute_normalized_deviations(data, indices)
def compute_normalized_deviations(self, data, indices): from xfel.merging.algorithms.error_model import compute_normalized_deviations return compute_normalized_deviations(data, indices)