def optimize(self, r_matrix, flag_plot=False):
xinp = flex.double([random.random() for i in range(len(r_matrix)*2)])
xinp_copy = xinp[:]
args = r_matrix
lh = lbfgs_handler(current_x=xinp, args=args)
x_set = np.array(lh.x).reshape((len(lh.x)/2,2))
if flag_plot:
import matplotlib.pyplot as plt
xinp_set = np.array(xinp_copy).reshape((len(xinp_copy)/2,2))
plt.subplot(2,1,1)
plt.scatter(xinp_set[:,0], xinp_set[:,1], s=10, marker='x', c='r')
plt.subplot(2,1,2)
plt.scatter(x_set[:,0], x_set[:,1], s=10, marker='x', c='r')
plt.show()
return x_set
def optimize(self, r_matrix, flag_plot=False):
xinp = flex.double([random.random() for i in range(len(r_matrix)*2)])
xinp_copy = xinp[:]
args = r_matrix
lh = lbfgs_handler(current_x=xinp, args=args)
x_set = np.array(lh.x).reshape((len(lh.x)//2,2))
if flag_plot:
import matplotlib.pyplot as plt
xinp_set = np.array(xinp_copy).reshape((len(xinp_copy)/2,2))
ax1 = plt.subplot(2,1,1)
ax1.scatter(xinp_set[:,0], xinp_set[:,1], s=10, marker='o', c='b')
ax1.set_xlim([-0.2, 1.2])
ax1.set_ylim([-0.2, 1.2])
ax2 = plt.subplot(2,1,2)
ax2.scatter(x_set[:,0], x_set[:,1], s=10, marker='o', c='b')
ax2.set_xlim([-0.2, 1.2])
ax2.set_ylim([-0.2, 1.2])
plt.show()
return x_set
def optimize_scalefactors(self, I_r_flex, observations_original,
wavelength, crystal_init_orientation,
alpha_angle, spot_pred_x_mm, spot_pred_y_mm,
iparams, pres_in, observations_non_polar,
detector_distance_mm, const_params):
ph = partiality_handler()
pr_d_min = iparams.postref.scale.d_min
pr_d_max = iparams.postref.scale.d_max
pr_sigma_min = iparams.postref.scale.sigma_min
pr_partiality_min = iparams.postref.scale.partiality_min
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,\
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,\
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()[:]
if pres_in is not None:
G, B, b0 = pres_in.G, pres_in.B, pres_in.B
else:
G, B, b0 = (1, 0, 0)
refine_mode = 'scale_factor'
xinp = flex.double([G, B])
args = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel,
spot_pred_y_mm_sel, detector_distance_mm, refine_mode,
const_params, b0, None, iparams)
lh = lbfgs_handler(current_x=xinp, args=args)
G_fin, B_fin = (lh.x[0], lh.x[1])
rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = const_params
two_theta = observations_original.two_theta(wavelength=wavelength)
sin_theta_over_lambda_sq = two_theta.sin_theta_over_lambda_sq().data()
uc = unit_cell((a, b, c, alpha, beta, gamma))
ph = partiality_handler()
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(
uc, rotx, roty, observations_original.indices(), ry, rz, r0, re,
voigt_nu, two_theta.data(), alpha_angle, wavelength,
crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm,
detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, G, B,
partiality_init, rs_init)
I_o_fin = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, G_fin, B_fin,
partiality_init, rs_init)
SE_of_the_estimate = standard_error_of_the_estimate(
I_r_flex, I_o_fin, 2)
R_sq = coefficient_of_determination(I_r_flex, I_o_fin) * 100
CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient()
CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient()
err_init = (I_r_flex - I_o_init) / observations_original.sigmas()
R_init = math.sqrt(flex.sum(err_init**2))
err_final = (I_r_flex - I_o_fin) / observations_original.sigmas()
R_final = math.sqrt(flex.sum(err_final**2))
R_xy_init = 0
R_xy_final = 0
CC_iso_init = 0
CC_iso_final = 0
return flex.double(list(lh.x)), (SE_of_the_estimate, R_sq, CC_init,
CC_final, R_init, R_final, R_xy_init,
R_xy_final, CC_iso_init, CC_iso_final)
def optimize(self, I_r_flex, observations_original, wavelength,
crystal_init_orientation, alpha_angle, spot_pred_x_mm,
spot_pred_y_mm, iparams, pres_in, observations_non_polar,
detector_distance_mm):
ph = partiality_handler()
lph = lbfgs_partiality_handler()
if iparams.postref.allparams.flag_on:
refine_steps = ['allparams']
else:
refine_steps = ['crystal_orientation']
if iparams.postref.reflecting_range.flag_on:
refine_steps.append('reflecting_range')
if iparams.postref.unit_cell.flag_on:
refine_steps.append('unit_cell')
#get miller array iso, if given.
miller_array_iso = None
#prepare data
pr_d_min = iparams.postref.allparams.d_min
pr_d_max = iparams.postref.allparams.d_max
pr_sigma_min = iparams.postref.allparams.sigma_min
pr_partiality_min = iparams.postref.allparams.partiality_min
pr_uc_tol = iparams.postref.allparams.uc_tolerance
cs = observations_original.crystal_symmetry().space_group(
).crystal_system()
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
#initialize values only in the first sub cycle and the first refine step.
spot_radius = ph.calc_spot_radius(
sqr(crystal_init_orientation.reciprocal_matrix()),
observations_original_sel.indices(), wavelength)
if pres_in is None:
ry, rz, r0, re, voigt_nu, rotx, roty = 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, 0.0, 0.0
#apply constrain on the unit cell using crystal system
uc_scale_inp = lph.prep_input(
observations_original.unit_cell().parameters(), cs)
uc_scale_constrained = lph.prep_output(uc_scale_inp, cs)
a, b, c, alpha, beta, gamma = uc_scale_constrained
const_params_scale = (rotx, roty, ry, rz, r0, re, voigt_nu, a, b,
c, alpha, beta, gamma)
xopt_scalefactors, stats = self.optimize_scalefactors(
I_r_flex, observations_original, wavelength,
crystal_init_orientation, alpha_angle, spot_pred_x_mm,
spot_pred_y_mm, iparams, pres_in, observations_non_polar,
detector_distance_mm, const_params_scale)
G, B = xopt_scalefactors
else:
G, B, ry, rz, r0, re, voigt_nu, rotx, roty = pres_in.G, pres_in.B, pres_in.ry, pres_in.rz, pres_in.r0, pres_in.re, pres_in.voigt_nu, 0.0, 0.0
a, b, c, alpha, beta, gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#filter by partiality
two_theta = observations_original_sel.two_theta(
wavelength=wavelength).data()
uc = unit_cell((a, b, c, alpha, beta, gamma))
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(
uc, rotx, roty, observations_original_sel.indices(), ry, rz, r0,
re, voigt_nu, two_theta, alpha_angle_sel, wavelength,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'partiality', [pr_partiality_min], observations_original_sel,
alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel,
partiality_in=partiality_init)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()[:]
#calculate initial residual_xy error
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell', const_params_uc, B,
miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
init_residual_xy_err = flex.sum(uc_params_err**2)
#calculate initial residual_pr error
const_params_all = (G, B)
xinp_all = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp_all.extend(lph.prep_input((a, b, c, alpha, beta, gamma), cs))
args_all = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'allparams', const_params_all, B,
miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
init_residual_err = flex.sum(all_params_err**2)
#keep in list
t_pr_list = [init_residual_err]
t_xy_list = [init_residual_xy_err]
refined_params_hist = [(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,
b, c, alpha, beta, gamma)]
txt_out = ''
for i_sub_cycle in range(iparams.n_postref_sub_cycle):
for j_refine_step in range(len(refine_steps)):
refine_mode = refine_steps[j_refine_step]
#prepare data
init_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,
b, c, alpha, beta, gamma)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.prepare_data_microcycle(refine_mode, iparams,
observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm,
I_r_flex, init_params, crystal_init_orientation,
wavelength, detector_distance_mm)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()
if refine_mode == 'crystal_orientation':
xinp = flex.double([rotx, roty])
const_params = (G, B, ry, rz, r0, re, voigt_nu, a, b, c,
alpha, beta, gamma)
elif refine_mode == 'reflecting_range':
xinp = flex.double([ry, rz, r0, re, voigt_nu])
const_params = (G, B, rotx, roty, a, b, c, alpha, beta,
gamma)
elif refine_mode == 'unit_cell':
xinp = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
const_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
elif refine_mode == 'allparams':
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(
lph.prep_input((a, b, c, alpha, beta, gamma), cs))
const_params = (G, B)
args = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, refine_mode, const_params, B,
miller_array_iso, iparams)
lh = lbfgs_handler(current_x=xinp, args=args)
xopt = flex.double(list(lh.x))
if refine_mode == 'crystal_orientation' or \
refine_mode == 'reflecting_range' or refine_mode == 'allparams':
current_residual_err = lh.f
#calculate residual_xy_error (for refine_mode = SF, CO, RR, and all params)
xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
if refine_mode == 'crystal_orientation':
rotx, roty = xopt
elif refine_mode == 'reflecting_range':
ry, rz, r0, re, voigt_nu = xopt
elif refine_mode == 'allparams':
rotx, roty, ry, rz, r0, re, voigt_nu = xopt[:7]
xinp_uc = xopt[7:]
a, b, c, alpha, beta, gamma = lph.prep_output(
xinp_uc, cs)
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re,
voigt_nu)
xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell',
const_params_uc, B, miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
current_residual_xy_err = flex.sum(uc_params_err**2)
elif refine_mode == 'unit_cell':
current_residual_xy_err = lh.f
xopt_uc = lph.prep_output(xopt, cs)
a, b, c, alpha, beta, gamma = xopt_uc
#check the unit-cell with the reference intensity
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(
lph.prep_input((a, b, c, alpha, beta, gamma), cs))
const_params_all = (G, B)
args_all = (I_r_true, observations_original_sel,
wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel,
spot_pred_y_mm_sel, detector_distance_mm,
'allparams', const_params_all, B,
miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
current_residual_err = flex.sum(all_params_err**2)
flag_success = False
if refine_mode == 'allparams':
#if allparams refinement, only check the post-refine target function
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append(
(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b,
c, alpha, beta, gamma))
flag_success = True
else:
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
if current_residual_xy_err < (t_xy_list[len(t_xy_list)-1] + \
(t_xy_list[len(t_xy_list)-1]*iparams.postref.residual_threshold_xy/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append(
(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,
b, c, alpha, beta, gamma))
flag_success = True
if flag_success is False:
G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = refined_params_hist[
len(refined_params_hist) - 1]
tmp_txt_out = refine_mode + ' %3.0f %6.4f %6.4f %6.4f %6.4f %10.8f %10.8f %10.8f %10.8f %10.8f %6.3f %6.3f %.4g %6.3f\n' % (
i_sub_cycle, G, B, rotx * 180 / math.pi, roty * 180 /
math.pi, ry, rz, r0, re, voigt_nu, a, c, t_pr_list[
len(t_pr_list) - 1], t_xy_list[len(t_pr_list) - 1])
txt_out += tmp_txt_out
#apply the refined parameters on the full (original) reflection set
two_theta = observations_original.two_theta(
wavelength=wavelength).data()
sin_theta_over_lambda_sq = observations_original.two_theta(
wavelength=wavelength).sin_theta_over_lambda_sq().data()
if pres_in is None:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
observations_original.unit_cell(),0.0, 0.0,observations_original.indices(),
0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, 1, 0,
partiality_init, rs_init)
else:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
pres_in.unit_cell,0.0, 0.0,observations_original.indices(),
pres_in.ry, pres_in.rz,pres_in.r0, pres_in.re, pres_in.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, pres_in.G,
pres_in.B, partiality_init, rs_init)
partiality_fin, delta_xy_fin, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(\
unit_cell((a,b,c,alpha,beta,gamma)),rotx, roty,observations_original.indices(),
ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength,crystal_init_orientation,
spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_fin = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, G, B,
partiality_fin, rs_fin)
SE_of_the_estimate = standard_error_of_the_estimate(
I_r_flex, I_o_fin, 13)
R_sq = coefficient_of_determination(I_r_flex, I_o_fin) * 100
CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient()
CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient()
err_init = (I_r_flex - I_o_init) / observations_original.sigmas()
R_init = math.sqrt(flex.sum(err_init**2))
err_final = (I_r_flex - I_o_fin) / observations_original.sigmas()
R_final = math.sqrt(flex.sum(err_final**2))
R_xy_init = math.sqrt(flex.sum(delta_xy_init**2))
R_xy_final = math.sqrt(flex.sum(delta_xy_fin**2))
if R_init < R_final or re > (iparams.gamma_e * 3):
CC_final = CC_init
R_final = R_init
R_xy_final = R_xy_init
if pres_in is None:
G, B, r0, ry, rz, re, rotx, roty = (1.0, 0.0, spot_radius, 0.0,
0.0, iparams.gamma_e, 0.0,
0.0)
a, b, c, alpha, beta, gamma = observations_original.unit_cell(
).parameters()
else:
G, B, r0, ry, rz, re, rotx, roty = (pres_in.G, pres_in.B,
pres_in.r0, pres_in.ry,
pres_in.rz, pres_in.re,
0.0, 0.0)
a, b, c, alpha, beta, gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#calculate CCiso if hklisoin is given
CC_iso_init, CC_iso_final = (0, 0)
if iparams.hklisoin is not None:
if miller_array_iso is not None:
from cctbx import miller
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_iso.indices(),
miller_indices=observations_non_polar.indices())
I_iso_match = flex.double([
miller_array_iso.data()[pair[0]]
for pair in matches.pairs()
])
I_o_init_match = flex.double(
[I_o_init[pair[1]] for pair in matches.pairs()])
I_o_fin_match = flex.double(
[I_o_fin[pair[1]] for pair in matches.pairs()])
CC_iso_init = flex.linear_correlation(
I_iso_match, I_o_init_match).coefficient()
CC_iso_final = flex.linear_correlation(
I_iso_match, I_o_fin_match).coefficient()
xopt = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha,
beta, gamma)
return xopt, (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init,
R_final, R_xy_init, R_xy_final, CC_iso_init,
CC_iso_final), len(I_ref_sel)
def optimize_scalefactors(self, I_r_flex, observations_original,
wavelength, crystal_init_orientation,
alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams,
pres_in, observations_non_polar, detector_distance_mm, const_params):
ph = partiality_handler()
pr_d_min = iparams.postref.scale.d_min
pr_d_max = iparams.postref.scale.d_max
pr_sigma_min = iparams.postref.scale.sigma_min
pr_partiality_min = iparams.postref.scale.partiality_min
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,\
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,\
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()[:]
if pres_in is not None:
G, B, b0 = pres_in.G, pres_in.B, pres_in.B
else:
G = flex.median(I_o_true)/flex.median(I_r_true)
B,b0 = (0,0)
if iparams.flag_apply_b_by_frame:
try:
from mod_util import mx_handler
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(iparams.n_residues)
observations_as_f = observations_original_sel.as_amplitude_array()
binner_template_asu = observations_as_f.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True)
G = wp.wilson_intensity_scale_factor*1e3
B = wp.wilson_b
except Exception:
pass
refine_mode = 'scale_factor'
xinp = flex.double([G,B])
args = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, refine_mode, const_params, b0, None, iparams)
lh = lbfgs_handler(current_x=xinp, args=args)
G_fin, B_fin = (lh.x[0], lh.x[1])
rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = const_params
two_theta = observations_original.two_theta(wavelength=wavelength)
sin_theta_over_lambda_sq = two_theta.sin_theta_over_lambda_sq().data()
uc = unit_cell((a,b,c,alpha,beta,gamma))
ph = partiality_handler()
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty,
observations_original.indices(),
ry, rz, r0, re, voigt_nu,
two_theta.data(),
alpha_angle,
wavelength,
crystal_init_orientation,
spot_pred_x_mm,
spot_pred_y_mm,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
G, B, partiality_init, rs_init)
I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
G_fin, B_fin, partiality_init, rs_init)
SE_of_the_estimate = standard_error_of_the_estimate(I_r_flex, I_o_fin, 2)
R_sq = coefficient_of_determination(I_r_flex,I_o_fin)*100
CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient()
CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient()
err_init = (I_r_flex - I_o_init)/observations_original.sigmas()
R_init = math.sqrt(flex.sum(err_init**2))
err_final = (I_r_flex - I_o_fin)/observations_original.sigmas()
R_final = math.sqrt(flex.sum(err_final**2))
R_xy_init = 0
R_xy_final = 0
CC_iso_init = 0
CC_iso_final = 0
return flex.double(list(lh.x)), (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final)
def optimize(self, I_r_flex, observations_original,
wavelength, crystal_init_orientation,
alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams,
pres_in, observations_non_polar, detector_distance_mm):
ph = partiality_handler()
lph = lbfgs_partiality_handler()
if iparams.postref.allparams.flag_on:
refine_steps = ['allparams']
else:
refine_steps = ['crystal_orientation']
if iparams.postref.reflecting_range.flag_on:
refine_steps.append('reflecting_range')
if iparams.postref.unit_cell.flag_on:
refine_steps.append('unit_cell')
#get miller array iso, if given.
miller_array_iso = None
#prepare data
pr_d_min = iparams.postref.allparams.d_min
pr_d_max = iparams.postref.allparams.d_max
pr_sigma_min = iparams.postref.allparams.sigma_min
pr_partiality_min = iparams.postref.allparams.partiality_min
pr_uc_tol = iparams.postref.allparams.uc_tolerance
cs = observations_original.crystal_symmetry().space_group().crystal_system()
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
#initialize values only in the first sub cycle and the first refine step.
spot_radius = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()),
observations_original_sel.indices(), wavelength)
if pres_in is None:
ry, rz, r0, re, voigt_nu, rotx, roty = 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, 0.0, 0.0
#apply constrain on the unit cell using crystal system
uc_scale_inp = lph.prep_input(observations_original.unit_cell().parameters(), cs)
uc_scale_constrained = lph.prep_output(uc_scale_inp, cs)
a,b,c,alpha,beta,gamma = uc_scale_constrained
const_params_scale = (rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)
xopt_scalefactors, stats = self.optimize_scalefactors(I_r_flex,
observations_original,
wavelength, crystal_init_orientation,
alpha_angle,
spot_pred_x_mm,
spot_pred_y_mm,
iparams,
pres_in,
observations_non_polar,
detector_distance_mm,
const_params_scale)
G, B = xopt_scalefactors
else:
G, B, ry, rz, r0, re, voigt_nu, rotx, roty = pres_in.G, pres_in.B, pres_in.ry, pres_in.rz, pres_in.r0, pres_in.re, pres_in.voigt_nu, 0.0 , 0.0
a,b,c,alpha,beta,gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#filter by partiality
two_theta = observations_original_sel.two_theta(wavelength=wavelength).data()
uc = unit_cell((a,b,c,alpha,beta,gamma))
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty,
observations_original_sel.indices(),
ry, rz, r0, re, voigt_nu, two_theta,
alpha_angle_sel, wavelength,
crystal_init_orientation,
spot_pred_x_mm_sel,
spot_pred_y_mm_sel,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'partiality', [pr_partiality_min], observations_original_sel,
alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel,
partiality_in=partiality_init)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()[:]
#calculate initial residual_xy error
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
init_residual_xy_err = flex.sum(uc_params_err**2)
#calculate initial residual_pr error
const_params_all= (G,B)
xinp_all = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp_all.extend(lph.prep_input((a,b,c,alpha,beta,gamma), cs))
args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
init_residual_err = flex.sum(all_params_err**2)
#keep in list
t_pr_list = [init_residual_err]
t_xy_list = [init_residual_xy_err]
refined_params_hist = [(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)]
txt_out = ''
for i_sub_cycle in range(iparams.n_postref_sub_cycle):
for j_refine_step in range(len(refine_steps)):
refine_mode = refine_steps[j_refine_step]
#prepare data
init_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.prepare_data_microcycle(refine_mode, iparams,
observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm,
I_r_flex, init_params, crystal_init_orientation,
wavelength, detector_distance_mm)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()
if refine_mode == 'crystal_orientation':
xinp = flex.double([rotx, roty])
const_params = (G, B, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)
elif refine_mode == 'reflecting_range':
xinp = flex.double([ry, rz, r0, re, voigt_nu])
const_params = (G, B, rotx, roty, a, b, c, alpha, beta, gamma)
elif refine_mode == 'unit_cell':
xinp = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
const_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
elif refine_mode == 'allparams':
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(lph.prep_input((a,b,c,alpha,beta,gamma), cs))
const_params = (G,B)
args=(I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, refine_mode, const_params, B, miller_array_iso, iparams)
lh = lbfgs_handler(current_x=xinp, args=args)
xopt = flex.double(list(lh.x))
if refine_mode == 'crystal_orientation' or \
refine_mode == 'reflecting_range' or refine_mode == 'allparams':
current_residual_err = lh.f
#calculate residual_xy_error (for refine_mode = SF, CO, RR, and all params)
xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
if refine_mode == 'crystal_orientation':
rotx, roty = xopt
elif refine_mode == 'reflecting_range':
ry, rz, r0, re, voigt_nu = xopt
elif refine_mode == 'allparams':
rotx, roty, ry, rz, r0, re, voigt_nu = xopt[:7]
xinp_uc = xopt[7:]
a, b, c, alpha, beta, gamma = lph.prep_output(xinp_uc, cs)
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
current_residual_xy_err = flex.sum(uc_params_err**2)
elif refine_mode == 'unit_cell':
current_residual_xy_err = lh.f
xopt_uc = lph.prep_output(xopt, cs)
a, b, c, alpha, beta, gamma = xopt_uc
#check the unit-cell with the reference intensity
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(lph.prep_input((a, b, c, alpha, beta, gamma), cs))
const_params_all = (G,B)
args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
current_residual_err = flex.sum(all_params_err**2)
flag_success = False
if refine_mode == 'allparams':
#if allparams refinement, only check the post-refine target function
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append((G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma))
flag_success = True
else:
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
if current_residual_xy_err < (t_xy_list[len(t_xy_list)-1] + \
(t_xy_list[len(t_xy_list)-1]*iparams.postref.residual_threshold_xy/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append((G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma))
flag_success = True
if flag_success is False:
G,B,rotx,roty,ry,rz,r0,re,voigt_nu,a,b,c,alpha,beta,gamma = refined_params_hist[len(refined_params_hist)-1]
tmp_txt_out = refine_mode + ' %3.0f %6.4f %6.4f %6.4f %6.4f %10.8f %10.8f %10.8f %10.8f %10.8f %6.3f %6.3f %.4g %6.3f\n'%(i_sub_cycle,G,B,rotx*180/math.pi,roty*180/math.pi,ry,rz,r0,re,voigt_nu,a,c,t_pr_list[len(t_pr_list)-1],t_xy_list[len(t_pr_list)-1])
txt_out += tmp_txt_out
#apply the refined parameters on the full (original) reflection set
two_theta = observations_original.two_theta(wavelength=wavelength).data()
sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data()
if pres_in is None:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
observations_original.unit_cell(),0.0, 0.0,observations_original.indices(),
0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
1, 0, partiality_init, rs_init)
else:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
pres_in.unit_cell,0.0, 0.0,observations_original.indices(),
pres_in.ry, pres_in.rz,pres_in.r0, pres_in.re, pres_in.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
pres_in.G, pres_in.B, partiality_init, rs_init)
partiality_fin, delta_xy_fin, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(\
unit_cell((a,b,c,alpha,beta,gamma)),rotx, roty,observations_original.indices(),
ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength,crystal_init_orientation,
spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
G, B, partiality_fin, rs_fin)
SE_of_the_estimate = standard_error_of_the_estimate(I_r_flex,I_o_fin, 13)
R_sq = coefficient_of_determination(I_r_flex,I_o_fin)*100
CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient()
CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient()
err_init = (I_r_flex - I_o_init)/observations_original.sigmas()
R_init = math.sqrt(flex.sum(err_init**2))
err_final = (I_r_flex - I_o_fin)/observations_original.sigmas()
R_final = math.sqrt(flex.sum(err_final**2))
R_xy_init = math.sqrt(flex.sum(delta_xy_init**2))
R_xy_final = math.sqrt(flex.sum(delta_xy_fin**2))
if R_init < R_final or re > (iparams.gamma_e * 10):
CC_final = CC_init
R_final = R_init
R_xy_final = R_xy_init
if pres_in is None:
G,B,r0,ry,rz,re,rotx,roty = (1.0,0.0,spot_radius,0.0,0.0,iparams.gamma_e,0.0,0.0)
a,b,c,alpha,beta,gamma = observations_original.unit_cell().parameters()
else:
G,B,r0,ry,rz,re,rotx,roty = (pres_in.G,pres_in.B,pres_in.r0,pres_in.ry,pres_in.rz,pres_in.re,0.0,0.0)
a,b,c,alpha,beta,gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#calculate CCiso if hklisoin is given
CC_iso_init,CC_iso_final = (0,0)
if iparams.hklisoin is not None:
if miller_array_iso is not None:
from cctbx import miller
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_iso.indices(),
miller_indices=observations_non_polar.indices())
I_iso_match = flex.double([miller_array_iso.data()[pair[0]] for pair in matches.pairs()])
I_o_init_match = flex.double([I_o_init[pair[1]] for pair in matches.pairs()])
I_o_fin_match = flex.double([I_o_fin[pair[1]] for pair in matches.pairs()])
CC_iso_init = flex.linear_correlation(I_iso_match, I_o_init_match).coefficient()
CC_iso_final = flex.linear_correlation(I_iso_match, I_o_fin_match).coefficient()
xopt = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,b,c,alpha,beta,gamma)
return xopt, (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final), len(I_ref_sel)