def search(self, rmax):
z_model = zernike_model(self.nlm_array, self.data.q,
rmax / self.fraction, self.nmax)
self.nn_array.load_coefs(self.nn, self.this_coef)
i_cal = z_model.calc_intensity(self.nn_array)
scale = i_cal[0]
i_cal = i_cal / (scale / self.int_scale)
if (self.smear):
new_i = smear_data(i_cal, self.data.q, self.delta_q)
self.this_score = flex.sum_sq(
(new_i - self.data.i[1:-2]) / self.weights[1:-2])
else:
self.this_score = flex.sum_sq((i_cal - self.data.i) / self.weights)
self.this_rmax = rmax
return self.this_score
def check_i_obs_vs_backup(O, work_params):
print "Current i_obs vs. backup:"
for im in O.array:
im.backup.reset_partialities(work_params, O.miller_indices)
b_obs = im.backup.extract_i_obs_est(work_params, O.miller_indices)
im.reset_partialities(work_params, O.miller_indices)
i_obs = im.extract_i_obs_est(work_params, O.miller_indices)
max_common_size = -1
max_cb_ci = None
for s in work_params.lattice_symmetry.group():
i_obs_cb = i_obs.change_basis(str(s))
cb, ci = b_obs.common_sets(other=i_obs_cb)
common_size = cb.indices().size()
if (max_common_size < common_size):
max_common_size = common_size
max_cb_ci = cb, ci
assert max_cb_ci is not None
cb, ci = max_cb_ci
from scitbx.array_family import flex
num = flex.sum(cb.data()*ci.data())
den = flex.sum_sq(cb.data())
if (den == 0): scale = None
else: scale = num / den
print " ", b_obs.indices().size(), i_obs.indices().size(), \
cb.indices().size(), scale
print
def check_i_obs_vs_backup(O, work_params):
print "Current i_obs vs. backup:"
for im in O.array:
im.backup.reset_partialities(work_params, O.miller_indices)
b_obs = im.backup.extract_i_obs_est(work_params, O.miller_indices)
im.reset_partialities(work_params, O.miller_indices)
i_obs = im.extract_i_obs_est(work_params, O.miller_indices)
max_common_size = -1
max_cb_ci = None
for s in work_params.lattice_symmetry.group():
i_obs_cb = i_obs.change_basis(str(s))
cb, ci = b_obs.common_sets(other=i_obs_cb)
common_size = cb.indices().size()
if (max_common_size < common_size):
max_common_size = common_size
max_cb_ci = cb, ci
assert max_cb_ci is not None
cb, ci = max_cb_ci
from scitbx.array_family import flex
num = flex.sum(cb.data()*ci.data())
den = flex.sum_sq(cb.data())
if (den == 0): scale = None
else: scale = num / den
print " ", b_obs.indices().size(), i_obs.indices().size(), \
cb.indices().size(), scale
print
def build_cluster_pair_info(O, other, work_params, reindexing_assistant):
from scitbx.array_family import flex
scale_max = work_params.scale_estimation_scale_max
assert scale_max > 0
scale_min = 1/scale_max
miis_i, esti_i = O.miis_perms[0], O.esti_perms[0]
result = []
for j_perm in xrange(len(reindexing_assistant.cb_ops)):
miis_j, esti_j = other.miis_perms[j_perm], other.esti_perms[j_perm]
i_seqs, j_seqs = miis_i.intersection_i_seqs(other=miis_j)
if (i_seqs.size() < 2):
return None
x = esti_i.select(i_seqs)
y = esti_j.select(j_seqs)
if (((x != 0) | (y != 0)).count(True) < 2):
return None
num = flex.sum(x*y)
den = flex.sum_sq(x)
if (num > den * scale_min and num < den * scale_max):
scale = num / den
rms = flex.mean_sq(x*scale-y)**0.5
result.append(perm_rms_info(n=x.size(), scale=scale, rms=rms))
else:
return None
result = perm_rms_list(array=result)
result.set_score()
return result
def build_cluster_pair_info(O, other, work_params, reindexing_assistant):
from scitbx.array_family import flex
scale_max = work_params.scale_estimation_scale_max
assert scale_max > 0
scale_min = 1/scale_max
miis_i, esti_i = O.miis_perms[0], O.esti_perms[0]
result = []
for j_perm in xrange(len(reindexing_assistant.cb_ops)):
miis_j, esti_j = other.miis_perms[j_perm], other.esti_perms[j_perm]
i_seqs, j_seqs = miis_i.intersection_i_seqs(other=miis_j)
if (i_seqs.size() < 2):
return None
x = esti_i.select(i_seqs)
y = esti_j.select(j_seqs)
if (((x != 0) | (y != 0)).count(True) < 2):
return None
num = flex.sum(x*y)
den = flex.sum_sq(x)
if (num > den * scale_min and num < den * scale_max):
scale = num / den
rms = flex.mean_sq(x*scale-y)**0.5
result.append(perm_rms_info(n=x.size(), scale=scale, rms=rms))
else:
return None
result = perm_rms_list(array=result)
result.set_score()
return result
def __init__(O, points, epsilon=1e-2):
"""\
Computation of Minimum-Volume Covering Ellipsoid using the Khachiyan Algorithm.
Based on a Python implementation by Raj Rajashankar (ANL, Nov 2011),
which in turn was based on a Matlab script by Nima Moshtagh (2009,
http://stackoverflow.com/questions/1768197/bounding-ellipse/1768440#1768440).
Caveats:
- center and radii are correct, but rotation may permute axes
- scales with the square of the number of points
"""
d = 3 # d is the dimension
n = points.size()
assert n > 0
#
from scitbx.array_family import flex
p = points.as_double()
p.reshape(flex.grid(n, 3))
p = p.matrix_transpose()
q = p.deep_copy()
q.resize(flex.grid(4, n), 1)
#
u = flex.double(n, 1/n)
umx = flex.double(flex.grid(n, n), 0)
#
err = epsilon + 1
while (err > epsilon):
umx.matrix_diagonal_set_in_place(u)
x_inv = q.matrix_multiply(umx).matrix_multiply_transpose(
q).matrix_inversion()
m = q.matrix_transpose_multiply(
x_inv).matrix_multiply(q).matrix_diagonal()
j = flex.max_index(m)
maximum = m[j]
ascent_step_size = (maximum-d-1)/((d+1)*(maximum-1))
new_u = (1 - ascent_step_size) * u
new_u[j] += ascent_step_size
err = flex.sum_sq(new_u - u)**0.5
u = new_u
#
center = p.matrix_multiply(u)
umx.matrix_diagonal_set_in_place(u)
t1 = p.matrix_multiply(umx).matrix_multiply_transpose(p)
t2 = center.matrix_outer_product(center)
a = (t1 - t2).matrix_inversion() / d
#
import scitbx.linalg.svd
svd = scitbx.linalg.svd.real(a, accumulate_u=False, accumulate_v=True)
size = 1.0/flex.sqrt(svd.sigma)
from scitbx import matrix
O.center = matrix.col(center)
O.radii = matrix.col(size)
O.rotation = matrix.sqr(svd.v)
def refinement_target(O, partiality_threshold):
assert O.miller_image_map.map is not None
from scitbx.array_family import flex
result_num = 0
result_den = 0
for iiis in O.miller_image_map.map:
estis = collect_estis(O.array, iiis, partiality_threshold)
if (estis.size() < 2): continue
i_obs_est = flex.mean(estis)
result_num += flex.sum_sq(estis - i_obs_est)
result_den += estis.size()
return result_num / max(1, result_den)
def __init__(O, points, epsilon=1e-2):
"""\
Computation of Minimum-Volume Covering Ellipsoid using the Khachiyan Algorithm.
Based on a Python implementation by Raj Rajashankar (ANL, Nov 2011),
which in turn was based on a Matlab script by Nima Moshtagh (2009,
http://stackoverflow.com/questions/1768197/bounding-ellipse/1768440#1768440).
Caveats:
- center and radii are correct, but rotation may permute axes
- scales with the square of the number of points
"""
d = 3 # d is the dimension
n = points.size()
assert n > 0
#
from scitbx.array_family import flex
p = points.as_double()
p.reshape(flex.grid(n, 3))
p = p.matrix_transpose()
q = p.deep_copy()
q.resize(flex.grid(4, n), 1)
#
u = flex.double(n, 1 / n)
umx = flex.double(flex.grid(n, n), 0)
#
err = epsilon + 1
while (err > epsilon):
umx.matrix_diagonal_set_in_place(u)
x_inv = q.matrix_multiply(umx).matrix_multiply_transpose(
q).matrix_inversion()
m = q.matrix_transpose_multiply(x_inv).matrix_multiply(
q).matrix_diagonal()
j = flex.max_index(m)
maximum = m[j]
ascent_step_size = (maximum - d - 1) / ((d + 1) * (maximum - 1))
new_u = (1 - ascent_step_size) * u
new_u[j] += ascent_step_size
err = flex.sum_sq(new_u - u)**0.5
u = new_u
#
center = p.matrix_multiply(u)
umx.matrix_diagonal_set_in_place(u)
t1 = p.matrix_multiply(umx).matrix_multiply_transpose(p)
t2 = center.matrix_outer_product(center)
a = (t1 - t2).matrix_inversion() / d
#
import scitbx.linalg.svd
svd = scitbx.linalg.svd.real(a, accumulate_u=False, accumulate_v=True)
size = 1.0 / flex.sqrt(svd.sigma)
from scitbx import matrix
O.center = matrix.col(center)
O.radii = matrix.col(size)
O.rotation = matrix.sqr(svd.v)
def refinement_target(O, partiality_threshold):
assert O.miller_image_map.map is not None
from scitbx.array_family import flex
result_num = 0
result_den = 0
for iiis in O.miller_image_map.map:
estis = collect_estis(O.array, iiis, partiality_threshold)
if (estis.size() < 2): continue
i_obs_est = flex.mean(estis)
result_num += flex.sum_sq(estis - i_obs_est)
result_den += estis.size()
return result_num / max(1, result_den)
def search(self, z_model, coefs, ntop, rmax):
scores = flex.double()
scales = flex.double()
fancy_weight_obj = error_model.two_stage_basic(self.data.q,
self.data.i,
self.data.s, 0.02, 0.75)
w = fancy_weight_obj.error_model_smear(rmax, 3)
w = flex.sqrt(w)
for coef in coefs:
self.nn_array.load_coefs(self.nn, coef)
i_cal = z_model.calc_intensity(self.nn_array)
scale = i_cal[0]
i_cal = i_cal / (scale / self.int_scale)
scale = self.approx_scale(self.data.i, i_cal, w)
if (self.smear):
new_i = smear_data(i_cal, self.data.q, self.delta_q)
score = flex.sum_sq(
(new_i - scale * self.data.i[1:-2]) / (scale * w[1:-2]))
#score = 1.0 - self.score_cc( new_i,self.data.i[1:-2], self.weights[1:-2] )
else:
score = flex.sum_sq(
(i_cal - self.data.i * scale) / (scale * w))
#score = 1.0 - self.score_cc( new_i,self.data.i,self.weights )
scores.append(score)
scales.append(scale)
top_hits = self.tops(scores, ntop)
top_scores = flex.double()
top_scales = flex.double()
for tt in top_hits:
top_scores.append(scores[tt])
top_scales.append(scales[tt])
return top_hits, top_scores, top_scales
def target(self, vector):
self.counter += 1
result = 0
length = flex.sum_sq(vector)
if (length > self.Rmax2):
result = 1e30
else:
new_coord = self.pdb.NMPerturb(self.modes, vector)
self.she_engine.engine.update_coord(new_coord, self.new_indx)
new_I = self.she_engine.engine.I()
var = self.expt_s
s, o = she.linear_fit(new_I[:5], self.expt_I[:5], var[:5])
result = flex.sum(
flex.pow2((self.expt_I - (s * new_I + o)) / self.expt_s))
# result = flex.sum_sq( (self.expt_I-new_I) /self.expt_s )
#print self.pdb.r, result
return result
def randomize_rotations(self):
hypersphere_point = flex.double(4)
self.rotations = [ None for i in xrange(len(self.species)) ]
for i in xrange(len(self.species)):
self.rotations[i] = [ None for j in xrange(self.species[i].n_copies) ]
for j in xrange(self.species[i].n_copies):
# pick random point on a 3-sphere
for k in xrange(hypersphere_point.size()):
hypersphere_point[k] = self.random.normalvariate(0.0,1.0)
hypersphere_point = hypersphere_point /\
math.sqrt(flex.sum_sq(hypersphere_point))
a = hypersphere_point[0]
b = hypersphere_point[1]
c = hypersphere_point[2]
d = hypersphere_point[3]
# convert quaternion to rotation matrix
self.rotations[i][j] =\
(a*a + b*b - c*c - d*d, 2*b*c - 2*a*d, 2*b*d + 2*a*c,
2*b*c + 2*a*d, a*a - b*b + c*c - d*d, 2*c*d - 2*a*b,
2*b*d - 2*a*c, 2*c*d + 2*a*b , a*a - b*b - c*c + d*d)
def search(self, z_model, coefs, ntop, rmax):
scores = flex.double()
scales = flex.double()
w = 1.0 ## temporary use
for coef in coefs:
self.nlm_array.load_coefs(self.nlm, coef)
calc_blq = z_model.get_all_blq(self.nlm_array)
scale = calc_blq[0]
calc_blq = calc_blq / scale
scale = self.approx_scale(self.data.blq, calc_blq, w)
score = flex.sum_sq(
(calc_blq - self.data.blq * scale) / (scale * w))
scores.append(score)
scales.append(scale)
top_hits = self.tops(scores, ntop)
top_scores = flex.double()
top_scales = flex.double()
for tt in top_hits:
top_scores.append(scores[tt])
top_scales.append(scales[tt])
return top_hits, top_scores, top_scales
def target(self, x): self.calc_data=self.calc_Cnm_from_Inm(x) score = flex.sum_sq( self.calc_data-self.target_data) return score/self.scale
def functional(self, x=None, f_x=None): if (f_x is None): f_x = self.f(x=x) return 0.5*flex.sum_sq(f_x)
def write_html_summary(fname, atom_group_summaries):
# Get template to be filled in
template = GIANT_HTML_ENV.get_template('summary_page.html')
# ===========================================================>
# Construct the data object to populate the template
output_data = {}
output_data['header'] = 'Structure Summary'
output_data['title'] = 'Structure Summary'
# # ===========================================================>
# # Progress Bars
# output_data['progress_bar'] = []
# output_data['progress_bar'].append({'title':'Dataset Summary', 'data':[]})
# output_data['progress_bar'][0]['data'].append({'text':'Interesting', 'colour':'success', 'size':100.0*num_interesting/len_data})
# output_data['progress_bar'][0]['data'].append({'text':'Not Interesting', 'colour':'info', 'size':100.0*num_not_interesting/len_data})
# output_data['progress_bar'][0]['data'].append({'text':'Not Analysed', 'colour':'warning', 'size':100.0*num_not_analysed/len_data})
# output_data['progress_bar'][0]['data'].append({'text':'Rejected', 'colour':'danger', 'size':100.0*num_rejected/len_data})
# ===========================================================>
# Tables
output_data['table'] = {}
output_data['table']['column_headings'] = [
'Num Atoms',
'Min BFactor',
'Max BFactor',
'Mean BFactor',
'RMS BFactor',
'Min BFactor-Z',
'Max BFactor-Z',
'Mean BFactor-Z',
'RMS BFactor-Z',
]
output_data['table']['rows'] = []
for ag_sum in atom_group_summaries:
# Extract the atom group info as a dictionary
ed_sum = ag_sum.edstats_scores
bfs = ag_sum.b_factors
bfs_stat = ag_sum.b_factors_statistics
bfz = ag_sum.b_factors_z
bfz_stat = ag_sum.b_factors_z_statistics
# colour choices - 'success', 'muted', 'danger'
# icon choices - 'ok', 'flag', 'remove'
columns = []
# Num Atoms
columns.append({'colour': 'default', 'message': round(bfs.size(), 1)})
# B-Factors
columns.append({
'colour': 'default',
'message': round(float(bfs_stat.min), 1)
})
columns.append({
'colour': 'default',
'message': round(float(bfs_stat.max), 1)
})
columns.append({
'colour': 'default',
'message': round(float(bfs_stat.mean), 1)
})
columns.append({
'colour':
'default',
'message':
round(float(flex.sum_sq(bfs) / bfs.size())**0.5, 1)
})
# B-Factors Z-Scores
columns.append({
'colour': 'default',
'message': round(float(bfz_stat.min), 2)
})
columns.append({
'colour': 'default',
'message': round(float(bfz_stat.max), 2)
})
columns.append({
'colour': 'default',
'message': round(float(bfz_stat.mean), 2)
})
if (flex.sum_sq(bfz) / bfz.size())**0.5 < 2.5:
columns.append({
'colour':
'default',
'message':
round(float(flex.sum_sq(bfz) / bfz.size())**0.5, 1)
})
else:
columns.append({
'colour':
'danger',
'icon':
'remove',
'message':
round((flex.sum_sq(bfz) / bfz.size())**0.5, 1)
})
row_message = '' if (flex.sum_sq(bfz)/bfz.size())**0.5 < 2.5 else \
'Check'
row_colour = 'success' if row_message == '' else \
'danger'
output_data['table']['rows'].append({
'heading':
ag_sum.atom_group.id_str(),
'message':
row_message,
'colour':
row_colour,
'columns':
columns
})
with open(fname, 'w') as out_html:
out_html.write(template.render(output_data))
def target(self, x): self.calc_data=self.calc_Cnm_from_Inm(x) score = flex.sum_sq( self.calc_data-self.target_data) return score/self.scale
def __init__(self,
function,
x0,
mu0=None,
tau=1.e-3,
eps_1=1.e-16,
eps_2=1.e-16,
mu_min=1.e-300,
k_max=1000):
self.function = function
x = x0
f_x = function.f(x=x)
number_of_function_evaluations = 1
self.f_x0 = f_x
fp = function.gradients(x=x, f_x=f_x)
number_of_gradient_evaluations = 1
number_of_hessian_evaluations = 0
number_of_cholesky_decompositions = 0
mu = mu0
nu = 2
k = 0
while (k < k_max):
if (flex.max(flex.abs(fp)) <= eps_1):
break
fdp = function.hessian(x=x)
number_of_hessian_evaluations += 1
if (mu is None):
mu = tau * flex.max(flex.abs(fdp))
if (mu == 0):
mu = fp.norm()/max(x.norm(), floating_point_epsilon_double**0.5)
fdp_plus_mu = fdp.deep_copy()
if (mu > mu_min):
fdp_plus_mu.matrix_diagonal_add_in_place(value=mu)
u = fdp_plus_mu.matrix_symmetric_as_packed_u()
gmw = scitbx.linalg.gill_murray_wright_cholesky_decomposition_in_place(u)
number_of_cholesky_decompositions += 1
h_dn = gmw.solve(b=-fp)
if (h_dn.norm() <= eps_2*(eps_2 + x.norm())):
break
x_new = x + h_dn
f_x_new = function.f(x=x_new)
number_of_function_evaluations += 1
fp_new = function.gradients(x=x_new, f_x=f_x_new)
number_of_gradient_evaluations += 1
f = flex.sum_sq(f_x)
fn = flex.sum_sq(f_x_new)
df = f - fn
accept = 0
if (df > 0):
accept = 1
dl = 0.5 * h_dn.dot(h_dn * mu - fp)
elif (fn <= f + abs(f)*(1+100*floating_point_epsilon_double)):
df = (fp + fp_new).dot(fp - fp_new)
if (df > 0):
accept = 2
if (accept != 0):
if (accept == 1 and dl > 0):
if (mu > mu_min):
mu *= max(1/3, 1 - (2*df/dl - 1)**3)
else:
mu = mu_min
nu = 2
else:
mu *= nu
nu *= 2
x = x_new
f_x = f_x_new
fp = fp_new
fdp = None
k += 1
else:
mu *= nu
nu *= 2
self.x_star = x
self.f_x_star = f_x
self.number_of_iterations = k
self.number_of_function_evaluations = number_of_function_evaluations
self.number_of_gradient_evaluations = number_of_gradient_evaluations
self.number_of_hessian_evaluations = number_of_hessian_evaluations
self.number_of_cholesky_decompositions = number_of_cholesky_decompositions
def functional(self, x=None, f_x=None): if (f_x is None): f_x = self.f(x=x) return 0.5*flex.sum_sq(f_x)
def target(self, r):
z_model = zernike_model(trivial_nlm_array, self.q_array, r, self.nmax)
calc_i = z_model.calc_intensity(self.nn_array)
calc_i = calc_i / calc_i[0]
return flex.sum_sq(self.ref_int - calc_i)
def __init__(self,
function,
x0,
mu0=None,
tau=1.e-3,
eps_1=1.e-16,
eps_2=1.e-16,
mu_min=1.e-300,
k_max=1000):
self.function = function
x = x0
f_x = function.f(x=x)
number_of_function_evaluations = 1
self.f_x0 = f_x
fp = function.gradients(x=x, f_x=f_x)
number_of_gradient_evaluations = 1
number_of_hessian_evaluations = 0
number_of_cholesky_decompositions = 0
mu = mu0
nu = 2
k = 0
while (k < k_max):
if (flex.max(flex.abs(fp)) <= eps_1):
break
fdp = function.hessian(x=x)
number_of_hessian_evaluations += 1
if (mu is None):
mu = tau * flex.max(flex.abs(fdp))
if (mu == 0):
mu = fp.norm() / max(x.norm(),
floating_point_epsilon_double**0.5)
fdp_plus_mu = fdp.deep_copy()
if (mu > mu_min):
fdp_plus_mu.matrix_diagonal_add_in_place(value=mu)
u = fdp_plus_mu.matrix_symmetric_as_packed_u()
gmw = scitbx.linalg.gill_murray_wright_cholesky_decomposition_in_place(
u)
number_of_cholesky_decompositions += 1
h_dn = gmw.solve(b=-fp)
if (h_dn.norm() <= eps_2 * (eps_2 + x.norm())):
break
x_new = x + h_dn
f_x_new = function.f(x=x_new)
number_of_function_evaluations += 1
fp_new = function.gradients(x=x_new, f_x=f_x_new)
number_of_gradient_evaluations += 1
f = flex.sum_sq(f_x)
fn = flex.sum_sq(f_x_new)
df = f - fn
accept = 0
if (df > 0):
accept = 1
dl = 0.5 * h_dn.dot(h_dn * mu - fp)
elif (fn <= f + abs(f) *
(1 + 100 * floating_point_epsilon_double)):
df = (fp + fp_new).dot(fp - fp_new)
if (df > 0):
accept = 2
if (accept != 0):
if (accept == 1 and dl > 0):
if (mu > mu_min):
mu *= max(1 / 3, 1 - (2 * df / dl - 1)**3)
else:
mu = mu_min
nu = 2
else:
mu *= nu
nu *= 2
x = x_new
f_x = f_x_new
fp = fp_new
fdp = None
k += 1
else:
mu *= nu
nu *= 2
self.x_star = x
self.f_x_star = f_x
self.number_of_iterations = k
self.number_of_function_evaluations = number_of_function_evaluations
self.number_of_gradient_evaluations = number_of_gradient_evaluations
self.number_of_hessian_evaluations = number_of_hessian_evaluations
self.number_of_cholesky_decompositions = number_of_cholesky_decompositions
def test_significance_of_group_of_z_values(vals):
"""Test the significance of a group of z-values - assumes sum-squared is distributed as a chi-squared"""
sum_sq = flex.sum_sq(flex.double(vals))
n = len(vals)
return chi_squared_test(sum_sq, n)
