def calc_partiality_mproc(i_sel, observations, crystal_init_orientation):
h = observations.indices()[i_sel]
partiality_c_set = flex.double()
a_star_init = sqr(crystal_init_orientation.reciprocal_matrix())
eh = energy_handler()
eh.get_energy_info(file_name_in_img, file_name_in_energy, pickle_filename)
#calculate spot_radisu (rs) from mean_wavelength
spot_radius = calc_spot_radius(a_star_init, observations.indices(), eh.mean_wavelength)
for wavelength, c_weight in zip(eh.wavelength_at_counts, eh.photon_counts_normalized):
ph = partiality_handler(wavelength, spot_radius)
p, dummy = ph.calc_partiality(a_star_init, h)
partiality_c_set.append(p*c_weight)
#correct intensity usinging single color
ph = partiality_handler(eh.mean_wavelength, spot_radius)
p_single, dummy = ph.calc_partiality(a_star_init, h)
I_p_single = observations.data()[i_sel]/p_single
sigI_p_single = observations.sigmas()[i_sel]/p_single
p_multi = sum(partiality_c_set)/sum(eh.photon_counts_normalized)
I_p_multi = observations.data()[i_sel]/p_multi
sigI_p_multi = observations.sigmas()[i_sel]/p_multi
return i_sel, h, p_single, I_p_single, sigI_p_single, p_multi, I_p_multi, sigI_p_multi
def prepare_data_microcycle(self, refine_mode, iparams,
observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm, I_r_flex,
init_params, crystal_init_orientation,
wavelength, detector_distance_mm):
#prepare data
if refine_mode == 'crystal_orientation':
pr_d_min = iparams.postref.crystal_orientation.d_min
pr_d_max = iparams.postref.crystal_orientation.d_max
pr_sigma_min = iparams.postref.crystal_orientation.sigma_min
pr_partiality_min = iparams.postref.crystal_orientation.partiality_min
pr_uc_tol = iparams.postref.unit_cell.uc_tolerance
elif refine_mode == 'reflecting_range':
pr_d_min = iparams.postref.reflecting_range.d_min
pr_d_max = iparams.postref.reflecting_range.d_max
pr_sigma_min = iparams.postref.reflecting_range.sigma_min
pr_partiality_min = iparams.postref.reflecting_range.partiality_min
pr_uc_tol = iparams.postref.unit_cell.uc_tolerance
elif refine_mode == 'unit_cell':
pr_d_min = iparams.postref.unit_cell.d_min
pr_d_max = iparams.postref.unit_cell.d_max
pr_sigma_min = iparams.postref.unit_cell.sigma_min
pr_partiality_min = iparams.postref.unit_cell.partiality_min
pr_uc_tol = iparams.postref.unit_cell.uc_tolerance
elif refine_mode == 'allparams':
pr_d_min = iparams.postref.allparams.d_min
pr_d_max = iparams.postref.allparams.d_max
pr_sigma_min = iparams.postref.allparams.sigma_min
pr_partiality_min = iparams.postref.allparams.partiality_min
pr_uc_tol = iparams.postref.unit_cell.uc_tolerance
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
#extract refined parameters
G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = init_params
#filter by partiality
two_theta = observations_original_sel.two_theta(
wavelength=wavelength).data()
uc = unit_cell((a, b, c, alpha, beta, gamma))
ph = partiality_handler()
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(
uc, rotx, roty, observations_original_sel.indices(), ry, rz, r0,
re, voigt_nu, two_theta, alpha_angle_sel, wavelength,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'partiality', [pr_partiality_min], observations_original_sel,
alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel,
partiality_in=partiality_init)
return observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel
def compute_cost_refine_crystal(self, I_ref, observations_original_sel, crystal_init_orientation,
wavelength, parameters):
G = parameters[0]
B_factor = parameters[1]
rotx = parameters[2]
roty = parameters[3]
ry = parameters[4]
rz = parameters[5]
I_obs = observations_original_sel.data()
sigI_obs = observations_original_sel.sigmas()
observations_original_sel_two_theta = observations_original_sel.two_theta(wavelength=wavelength)
two_theta = observations_original_sel_two_theta.data()
sin_theta_over_lambda_sq = observations_original_sel_two_theta.sin_theta_over_lambda_sq().data()
effective_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx
).rotate_thru((0,1,0),roty)
effective_a_star = sqr(effective_orientation.reciprocal_matrix())
ph = partiality_handler(wavelength, 0)
partiality = ph.calc_partiality_anisotropy_set(effective_a_star, observations_original_sel.indices(), ry, rz, two_theta)
excursions = (((G * (flex.exp(-2*B_factor*sin_theta_over_lambda_sq)) * I_obs)/partiality) - I_ref) / sigI_obs
return excursions
def jacobian_callable(pfh,current_values):
ry = current_values[0]
rz = current_values[1]
G = scale_factors[0]
B_factor = scale_factors[1]
rotx = rotations[0]
roty = rotations[1]
I_obs = observations_original_sel.data()
sigI_obs = observations_original_sel.sigmas()
observations_original_sel_two_theta = observations_original_sel.two_theta(wavelength=wavelength)
two_theta = observations_original_sel_two_theta.data()
sin_theta_over_lambda_sq = observations_original_sel_two_theta.sin_theta_over_lambda_sq().data()
delta_r = 0.00000001
#0. Calculate current partiality based on current rotx
crystal_current_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx
).rotate_thru((0,1,0),roty)
a_star = sqr(crystal_current_orientation.reciprocal_matrix())
ph = partiality_handler(wavelength, 0)
pDg_pDry = flex.double()
pDg_pDrz = flex.double()
for j in range(len(observations_original_sel.indices())):
miller_index = observations_original_sel.indices()[j]
p, rh, rs = ph.calc_partiality_anisotropy(a_star, miller_index, ry, rz, two_theta[j])
pDg_pDP = (-1 * G * (math.exp(-2*B_factor*sin_theta_over_lambda_sq[j])) * I_obs[j])/(sigI_obs[j]*(p**2))
pDP_pDrs = (4*rs*(rh**2))/(((2*(rh**2))+(rs**2))**2)
rs_as_k = math.sqrt(((ry * math.cos(two_theta[j]))**2)+((rz * math.sin(two_theta[j]))**2))
pDrs_pDry = ry * (math.cos(two_theta[j])**2)/ rs_as_k
pDrs_pDrz = rz * (math.sin(two_theta[j])**2)/ rs_as_k
pDg_pDry.append(pDg_pDP * pDP_pDrs * pDrs_pDry)
pDg_pDrz.append(pDg_pDP * pDP_pDrs * pDrs_pDrz)
"""
#checking partial derivatives
g_now = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx,roty,ry,rz))
g_delta_ry = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx,roty,ry+delta_r,rz))
g_delta_rz = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx,roty,ry,rz+delta_r))
pDg_pDry_fd = (g_delta_ry - g_now)/ delta_r
pDg_pDrz_fd = (g_delta_rz - g_now)/ delta_r
print sum(flex.abs(pDg_pDry_fd)), sum(flex.abs(pDg_pDry_fd-pDg_pDry))
print sum(flex.abs(pDg_pDrz_fd)), sum(flex.abs(pDg_pDrz_fd-pDg_pDrz))
print
"""
return pDg_pDry, pDg_pDrz
def calc_partiality_mproc(calc_partiality_arg, frames):
calc_partiality_arg_tag = calc_partiality_arg.split(':')
frame_no = int(calc_partiality_arg_tag[0])
ry_shift = float(calc_partiality_arg_tag[1])
pickle_filename = frames[frame_no]
trial_results = pickle.load(open(pickle_filename,"rb"))
crystal_init_orientation = trial_results["current_orientation"][0]
wavelength = trial_results["wavelength"]
observations = trial_results["observations"][0]
crystal_pointgroup = trial_results["pointgroup"]
unit_cell = trial_results["current_orientation"][0].unit_cell()
target_unit_cell = unit_cell.parameters()
a_star_init = sqr(crystal_init_orientation.reciprocal_matrix())
eh = energy_handler()
eh.get_energy_info(file_name_in_img, file_name_in_energy, pickle_filename)
#calc spot_radius y and z component
spot_radius = calc_spot_radius(a_star_init, observations.indices(), wavelength)
ry = spot_radius + ry_shift
rz = ry * 0.5
#calc alpha_angle
two_theta = observations.two_theta(wavelength=wavelength).data()
ph = partiality_handler(wavelength, spot_radius)
I_p_aniso = flex.double()
sigI_p_aniso = flex.double()
I_p = flex.double()
sigI_p = flex.double()
for i_sel in range(len(observations.indices())):
h = observations.indices()[i_sel]
p, dummy = ph.calc_partiality(a_star_init, h)
I_p.append(observations.data()[i_sel]/ p)
sigI_p.append(observations.sigmas()[i_sel]/ p)
alpha_angle = two_theta[i_sel]
p_aniso, rs_aniso = ph.calc_partiality_anisotropy(a_star_init, h, ry, rz, alpha_angle)
I_p_aniso.append(observations.data()[i_sel]/ p_aniso)
sigI_p_aniso.append(observations.sigmas()[i_sel]/ p_aniso)
observations_p = observations.customized_copy(data=I_p, sigmas=sigI_p)
observations_p_aniso = observations.customized_copy(data=I_p_aniso, sigmas=sigI_p_aniso)
return frame_no, observations_p, observations_p_aniso, (ry, rz, spot_radius)
def func_partiality(x, *args):
miller_index = args[0]
crystal_rotation_matrix = args[1]
wavelength = args[2]
bragg_angle, alpha_angle = args[3]
const_params = args[4]
fmode = args[5]
G, B, rotx, roty, ry, rz, re, a, b, c, alpha, beta, gamma = const_params
if fmode == 'G':
G = x
elif fmode== 'B':
B = x
elif fmode== 'rotx':
rotx = x
elif fmode== 'roty':
roty = x
elif fmode== 'ry':
ry = x
elif fmode== 'rz':
rz = x
elif fmode== 're':
re = x
elif fmode== 'a':
a = x
elif fmode== 'b':
b = x
elif fmode== 'c':
c = x
elif fmode== 'alpha':
alpha = x
elif fmode== 'beta':
beta = x
elif fmode== 'gamma':
gamma = x
uc = unit_cell((a,b,c,alpha,beta,gamma))
crystal_init_orientation = get_crystal_orientation(uc.orthogonalization_matrix(), crystal_rotation_matrix)
crystal_orientation_model = crystal_init_orientation.rotate_thru((1,0,0), rotx
).rotate_thru((0,1,0), roty)
a_star_model = sqr(crystal_orientation_model.reciprocal_matrix())
ph = partiality_handler(wavelength, 0)
partiality, dummy, dummy = ph.calc_partiality_anisotropy(a_star_model, miller_index, ry, rz, re, bragg_angle, alpha_angle)
return partiality
def calc_full_observations(self,
observations_type='asymmetric_unit',
flag_apply_partiality=False):
"""
calculate partiality and full observations (miller array)
"""
if True:
if self.init_type == 'from_file':
partiality_hdlr = partiality_handler(self.iparams)
self.partiality, dum0, self.rh_set, self.rs_set = partiality_hdlr.calc_partiality_anisotropy_set(
self)
b_cart = sqr((self.b11, self.b12, self.b13, self.b12, self.b22,
self.b23, self.b13, self.b23, self.b33))
dw = self.calc_debye_waller_factor(self.observations, b_cart)
if flag_apply_partiality:
partiality = self.partiality[:]
else:
partiality = flex.double([1] *
len(self.observations.data()))
G_set = flex.double([self.G] * len(self.observations.data()))
i_lowres = self.observations.resolution_filter_selection(
d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
G_set.set_selected(i_lowres, self.G_lowres)
dw.set_selected(i_lowres, 1.0)
I_full = flex.double(self.observations.data() /
(G_set * dw * partiality))
sigI_full = flex.double(self.observations.sigmas() /
(G_set * dw * partiality))
elif self.init_type == 'from_merge':
I_full = self.observations.data()[:]
sigI_full = self.observations.sigmas()[:]
if observations_type == 'asymmetric_unit':
observations_full = self.observations.customized_copy(
data=I_full, sigmas=sigI_full)
elif observations_type == 'original':
observations_full = self.observations_original.customized_copy(
data=I_full, sigmas=sigI_full)
return observations_full
def compute_cost(self, I_ref, I_obs, sigI_obs, crystal_init_orientation,
miller_indices_ori, wavelength, current_values):
G = current_values[0]
rotx = current_values[1]
roty = current_values[2]
rotz = current_values[3]
rs = current_values[4]
m = len(I_ref)
effective_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx
).rotate_thru((0,1,0),roty
).rotate_thru((0,0,1),rotz)
effective_a_star = sqr(effective_orientation.reciprocal_matrix())
ph = partiality_handler(wavelength, rs)
partiality = flex.double([ph.calc_partiality(effective_a_star, miller_index)[0] for miller_index in miller_indices_ori])
J = flex.sum(((((G * I_obs)/partiality)- I_ref)/ sigI_obs)**2)/(2*m)
return J
def fvec_callable(pfh,current_values):
rotx = current_values[0]
roty = current_values[1]
ry = spot_radius
rz = spot_radius
G = scale_factors[0]
B_factor = scale_factors[1]
I_obs = observations_original_sel.data()
sigI_obs = observations_original_sel.sigmas()
observations_original_sel_two_theta = observations_original_sel.two_theta(wavelength=wavelength)
two_theta = observations_original_sel_two_theta.data()
sin_theta_over_lambda_sq = observations_original_sel_two_theta.sin_theta_over_lambda_sq().data()
effective_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx
).rotate_thru((0,1,0),roty)
effective_a_star = sqr(effective_orientation.reciprocal_matrix())
ph = partiality_handler(wavelength, 0)
partiality = ph.calc_partiality_anisotropy_set(effective_a_star, observations_original_sel.indices(), ry, rz, two_theta)
excursions = (((G * (flex.exp(-2*B_factor*sin_theta_over_lambda_sq)) * I_obs)/partiality) - I_ref) / sigI_obs
corr_now, slope_now = get_overall_correlation((G * (flex.exp(-2*B_factor*sin_theta_over_lambda_sq)) * I_obs)/partiality, I_ref)
"""
print "ROTATION G=%5.3f B_factor=%5.3f rotx=%6.5f roty=%6.5f ry=%6.5f rz=%6.5f J=%6.3f cc=%6.3f slope=%6.3f p_mean=%6.3f"% \
(G, B_factor, rotx*180/math.pi, roty*180/math.pi, ry, rz, sum(excursions**2), corr_now, slope_now, flex.mean(partiality))
plt.scatter(I_ref,(G * (flex.exp(-2*B_factor*sin_theta_over_lambda_sq)) * I_obs)/partiality,s=10, marker='x', c='r')
plt.title('J=%6.5f CC=%6.5f Slope=%6.5f'%(sum(excursions**2), corr_now, slope_now))
plt.xlabel('Reference intensity')
plt.ylabel('Observed intensity (scaled)')
plt.show()
"""
return excursions
assume_index_matching=True)
print 'Overall R-factor:', r1_factor
#calculate r_factor for partiality-corrected scaled data
pixel_size_mm = 0.079346
crystal_init_orientation = observations_pickle["current_orientation"][0]
wavelength = observations_pickle["wavelength"]
detector_distance_mm = observations_pickle['distance']
mm_predictions = pixel_size_mm * (
observations_pickle['mapped_predictions'][0])
xbeam = observations_pickle["xbeam"]
ybeam = observations_pickle["ybeam"]
alpha_angle_obs = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \
for pred in mm_predictions])
spot_pred_x_mm = flex.double([pred[0] - xbeam for pred in mm_predictions])
spot_pred_y_mm = flex.double([pred[1] - ybeam for pred in mm_predictions])
ph = partiality_handler()
r0 = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()),
observations.indices(), wavelength)
ry, rz, re, voigt_nu, rotx, roty = (0, 0, 0.003, 0.5, 0, 0)
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
crystal_init_orientation.unit_cell(),
rotx, roty, observations.indices(),
ry, rz, r0, re, voigt_nu,
two_theta, alpha_angle_obs, wavelength,
crystal_init_orientation,
spot_pred_x_mm, spot_pred_y_mm,
detector_distance_mm, "Voigt",
False)
I_full = observations.data() / partiality_init
sigI_full = observations.sigmas() / partiality_init
obs_full = observations.customized_copy(data=I_full, sigmas=sigI_full)
def optimize_scalefactors(self, I_r_flex, observations_original,
wavelength, crystal_init_orientation,
alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams,
pres_in, observations_non_polar, detector_distance_mm, const_params):
ph = partiality_handler()
pr_d_min = iparams.postref.scale.d_min
pr_d_max = iparams.postref.scale.d_max
pr_sigma_min = iparams.postref.scale.sigma_min
pr_partiality_min = iparams.postref.scale.partiality_min
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,\
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,\
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()[:]
if pres_in is not None:
G, B, b0 = pres_in.G, pres_in.B, pres_in.B
else:
G = flex.median(I_o_true)/flex.median(I_r_true)
B,b0 = (0,0)
if iparams.flag_apply_b_by_frame:
try:
from mod_util import mx_handler
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(iparams.n_residues)
observations_as_f = observations_original_sel.as_amplitude_array()
binner_template_asu = observations_as_f.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True)
G = wp.wilson_intensity_scale_factor*1e3
B = wp.wilson_b
except Exception:
pass
refine_mode = 'scale_factor'
xinp = flex.double([G,B])
args = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, refine_mode, const_params, b0, None, iparams)
lh = lbfgs_handler(current_x=xinp, args=args)
G_fin, B_fin = (lh.x[0], lh.x[1])
rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = const_params
two_theta = observations_original.two_theta(wavelength=wavelength)
sin_theta_over_lambda_sq = two_theta.sin_theta_over_lambda_sq().data()
uc = unit_cell((a,b,c,alpha,beta,gamma))
ph = partiality_handler()
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty,
observations_original.indices(),
ry, rz, r0, re, voigt_nu,
two_theta.data(),
alpha_angle,
wavelength,
crystal_init_orientation,
spot_pred_x_mm,
spot_pred_y_mm,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
G, B, partiality_init, rs_init)
I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
G_fin, B_fin, partiality_init, rs_init)
SE_of_the_estimate = standard_error_of_the_estimate(I_r_flex, I_o_fin, 2)
R_sq = coefficient_of_determination(I_r_flex,I_o_fin)*100
CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient()
CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient()
err_init = (I_r_flex - I_o_init)/observations_original.sigmas()
R_init = math.sqrt(flex.sum(err_init**2))
err_final = (I_r_flex - I_o_fin)/observations_original.sigmas()
R_final = math.sqrt(flex.sum(err_final**2))
R_xy_init = 0
R_xy_final = 0
CC_iso_init = 0
CC_iso_final = 0
return flex.double(list(lh.x)), (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final)
def optimize(self, I_r_flex, observations_original,
wavelength, crystal_init_orientation,
alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams,
pres_in, observations_non_polar, detector_distance_mm):
ph = partiality_handler()
lph = lbfgs_partiality_handler()
if iparams.postref.allparams.flag_on:
refine_steps = ['allparams']
else:
refine_steps = ['crystal_orientation']
if iparams.postref.reflecting_range.flag_on:
refine_steps.append('reflecting_range')
if iparams.postref.unit_cell.flag_on:
refine_steps.append('unit_cell')
#get miller array iso, if given.
miller_array_iso = None
#prepare data
pr_d_min = iparams.postref.allparams.d_min
pr_d_max = iparams.postref.allparams.d_max
pr_sigma_min = iparams.postref.allparams.sigma_min
pr_partiality_min = iparams.postref.allparams.partiality_min
pr_uc_tol = iparams.postref.allparams.uc_tolerance
cs = observations_original.crystal_symmetry().space_group().crystal_system()
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
#initialize values only in the first sub cycle and the first refine step.
spot_radius = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()),
observations_original_sel.indices(), wavelength)
if pres_in is None:
ry, rz, r0, re, voigt_nu, rotx, roty = 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, 0.0, 0.0
#apply constrain on the unit cell using crystal system
uc_scale_inp = lph.prep_input(observations_original.unit_cell().parameters(), cs)
uc_scale_constrained = lph.prep_output(uc_scale_inp, cs)
a,b,c,alpha,beta,gamma = uc_scale_constrained
const_params_scale = (rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)
xopt_scalefactors, stats = self.optimize_scalefactors(I_r_flex,
observations_original,
wavelength, crystal_init_orientation,
alpha_angle,
spot_pred_x_mm,
spot_pred_y_mm,
iparams,
pres_in,
observations_non_polar,
detector_distance_mm,
const_params_scale)
G, B = xopt_scalefactors
else:
G, B, ry, rz, r0, re, voigt_nu, rotx, roty = pres_in.G, pres_in.B, pres_in.ry, pres_in.rz, pres_in.r0, pres_in.re, pres_in.voigt_nu, 0.0 , 0.0
a,b,c,alpha,beta,gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#filter by partiality
two_theta = observations_original_sel.two_theta(wavelength=wavelength).data()
uc = unit_cell((a,b,c,alpha,beta,gamma))
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty,
observations_original_sel.indices(),
ry, rz, r0, re, voigt_nu, two_theta,
alpha_angle_sel, wavelength,
crystal_init_orientation,
spot_pred_x_mm_sel,
spot_pred_y_mm_sel,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'partiality', [pr_partiality_min], observations_original_sel,
alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel,
partiality_in=partiality_init)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()[:]
#calculate initial residual_xy error
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
init_residual_xy_err = flex.sum(uc_params_err**2)
#calculate initial residual_pr error
const_params_all= (G,B)
xinp_all = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp_all.extend(lph.prep_input((a,b,c,alpha,beta,gamma), cs))
args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
init_residual_err = flex.sum(all_params_err**2)
#keep in list
t_pr_list = [init_residual_err]
t_xy_list = [init_residual_xy_err]
refined_params_hist = [(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)]
txt_out = ''
for i_sub_cycle in range(iparams.n_postref_sub_cycle):
for j_refine_step in range(len(refine_steps)):
refine_mode = refine_steps[j_refine_step]
#prepare data
init_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.prepare_data_microcycle(refine_mode, iparams,
observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm,
I_r_flex, init_params, crystal_init_orientation,
wavelength, detector_distance_mm)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()
if refine_mode == 'crystal_orientation':
xinp = flex.double([rotx, roty])
const_params = (G, B, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)
elif refine_mode == 'reflecting_range':
xinp = flex.double([ry, rz, r0, re, voigt_nu])
const_params = (G, B, rotx, roty, a, b, c, alpha, beta, gamma)
elif refine_mode == 'unit_cell':
xinp = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
const_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
elif refine_mode == 'allparams':
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(lph.prep_input((a,b,c,alpha,beta,gamma), cs))
const_params = (G,B)
args=(I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, refine_mode, const_params, B, miller_array_iso, iparams)
lh = lbfgs_handler(current_x=xinp, args=args)
xopt = flex.double(list(lh.x))
if refine_mode == 'crystal_orientation' or \
refine_mode == 'reflecting_range' or refine_mode == 'allparams':
current_residual_err = lh.f
#calculate residual_xy_error (for refine_mode = SF, CO, RR, and all params)
xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
if refine_mode == 'crystal_orientation':
rotx, roty = xopt
elif refine_mode == 'reflecting_range':
ry, rz, r0, re, voigt_nu = xopt
elif refine_mode == 'allparams':
rotx, roty, ry, rz, r0, re, voigt_nu = xopt[:7]
xinp_uc = xopt[7:]
a, b, c, alpha, beta, gamma = lph.prep_output(xinp_uc, cs)
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
current_residual_xy_err = flex.sum(uc_params_err**2)
elif refine_mode == 'unit_cell':
current_residual_xy_err = lh.f
xopt_uc = lph.prep_output(xopt, cs)
a, b, c, alpha, beta, gamma = xopt_uc
#check the unit-cell with the reference intensity
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(lph.prep_input((a, b, c, alpha, beta, gamma), cs))
const_params_all = (G,B)
args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
current_residual_err = flex.sum(all_params_err**2)
flag_success = False
if refine_mode == 'allparams':
#if allparams refinement, only check the post-refine target function
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append((G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma))
flag_success = True
else:
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
if current_residual_xy_err < (t_xy_list[len(t_xy_list)-1] + \
(t_xy_list[len(t_xy_list)-1]*iparams.postref.residual_threshold_xy/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append((G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma))
flag_success = True
if flag_success is False:
G,B,rotx,roty,ry,rz,r0,re,voigt_nu,a,b,c,alpha,beta,gamma = refined_params_hist[len(refined_params_hist)-1]
tmp_txt_out = refine_mode + ' %3.0f %6.4f %6.4f %6.4f %6.4f %10.8f %10.8f %10.8f %10.8f %10.8f %6.3f %6.3f %.4g %6.3f\n'%(i_sub_cycle,G,B,rotx*180/math.pi,roty*180/math.pi,ry,rz,r0,re,voigt_nu,a,c,t_pr_list[len(t_pr_list)-1],t_xy_list[len(t_pr_list)-1])
txt_out += tmp_txt_out
#apply the refined parameters on the full (original) reflection set
two_theta = observations_original.two_theta(wavelength=wavelength).data()
sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data()
if pres_in is None:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
observations_original.unit_cell(),0.0, 0.0,observations_original.indices(),
0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
1, 0, partiality_init, rs_init)
else:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
pres_in.unit_cell,0.0, 0.0,observations_original.indices(),
pres_in.ry, pres_in.rz,pres_in.r0, pres_in.re, pres_in.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
pres_in.G, pres_in.B, partiality_init, rs_init)
partiality_fin, delta_xy_fin, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(\
unit_cell((a,b,c,alpha,beta,gamma)),rotx, roty,observations_original.indices(),
ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength,crystal_init_orientation,
spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
G, B, partiality_fin, rs_fin)
SE_of_the_estimate = standard_error_of_the_estimate(I_r_flex,I_o_fin, 13)
R_sq = coefficient_of_determination(I_r_flex,I_o_fin)*100
CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient()
CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient()
err_init = (I_r_flex - I_o_init)/observations_original.sigmas()
R_init = math.sqrt(flex.sum(err_init**2))
err_final = (I_r_flex - I_o_fin)/observations_original.sigmas()
R_final = math.sqrt(flex.sum(err_final**2))
R_xy_init = math.sqrt(flex.sum(delta_xy_init**2))
R_xy_final = math.sqrt(flex.sum(delta_xy_fin**2))
if R_init < R_final or re > (iparams.gamma_e * 10):
CC_final = CC_init
R_final = R_init
R_xy_final = R_xy_init
if pres_in is None:
G,B,r0,ry,rz,re,rotx,roty = (1.0,0.0,spot_radius,0.0,0.0,iparams.gamma_e,0.0,0.0)
a,b,c,alpha,beta,gamma = observations_original.unit_cell().parameters()
else:
G,B,r0,ry,rz,re,rotx,roty = (pres_in.G,pres_in.B,pres_in.r0,pres_in.ry,pres_in.rz,pres_in.re,0.0,0.0)
a,b,c,alpha,beta,gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#calculate CCiso if hklisoin is given
CC_iso_init,CC_iso_final = (0,0)
if iparams.hklisoin is not None:
if miller_array_iso is not None:
from cctbx import miller
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_iso.indices(),
miller_indices=observations_non_polar.indices())
I_iso_match = flex.double([miller_array_iso.data()[pair[0]] for pair in matches.pairs()])
I_o_init_match = flex.double([I_o_init[pair[1]] for pair in matches.pairs()])
I_o_fin_match = flex.double([I_o_fin[pair[1]] for pair in matches.pairs()])
CC_iso_init = flex.linear_correlation(I_iso_match, I_o_init_match).coefficient()
CC_iso_final = flex.linear_correlation(I_iso_match, I_o_fin_match).coefficient()
xopt = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,b,c,alpha,beta,gamma)
return xopt, (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final), len(I_ref_sel)
def prepare_data_microcycle(self, refine_mode, iparams,
observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm,
I_r_flex, init_params, crystal_init_orientation,
wavelength, detector_distance_mm):
#prepare data
if refine_mode == 'crystal_orientation':
pr_d_min = iparams.postref.crystal_orientation.d_min
pr_d_max = iparams.postref.crystal_orientation.d_max
pr_sigma_min = iparams.postref.crystal_orientation.sigma_min
pr_partiality_min = iparams.postref.crystal_orientation.partiality_min
pr_uc_tol = iparams.postref.unit_cell.uc_tolerance
elif refine_mode == 'reflecting_range':
pr_d_min = iparams.postref.reflecting_range.d_min
pr_d_max = iparams.postref.reflecting_range.d_max
pr_sigma_min = iparams.postref.reflecting_range.sigma_min
pr_partiality_min = iparams.postref.reflecting_range.partiality_min
pr_uc_tol = iparams.postref.unit_cell.uc_tolerance
elif refine_mode == 'unit_cell':
pr_d_min = iparams.postref.unit_cell.d_min
pr_d_max = iparams.postref.unit_cell.d_max
pr_sigma_min = iparams.postref.unit_cell.sigma_min
pr_partiality_min = iparams.postref.unit_cell.partiality_min
pr_uc_tol = iparams.postref.unit_cell.uc_tolerance
elif refine_mode == 'allparams':
pr_d_min = iparams.postref.allparams.d_min
pr_d_max = iparams.postref.allparams.d_max
pr_sigma_min = iparams.postref.allparams.sigma_min
pr_partiality_min = iparams.postref.allparams.partiality_min
pr_uc_tol = iparams.postref.unit_cell.uc_tolerance
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
#extract refined parameters
G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = init_params
#filter by partiality
two_theta = observations_original_sel.two_theta(wavelength=wavelength).data()
uc = unit_cell((a,b,c,alpha,beta,gamma))
ph = partiality_handler()
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty,
observations_original_sel.indices(),
ry, rz, r0, re, voigt_nu, two_theta,
alpha_angle_sel, wavelength,
crystal_init_orientation,
spot_pred_x_mm_sel,
spot_pred_y_mm_sel,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'partiality', [pr_partiality_min], observations_original_sel,
alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel,
partiality_in=partiality_init)
return observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel
def scale_frame_by_mean_I(self, frame_no, pickle_filename, iparams,
mean_of_mean_I, avg_mode):
observations_pickle = read_frame(pickle_filename)
pickle_filepaths = pickle_filename.split('/')
img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1]
inputs, txt_organize_input = self.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pickle_filename)
txt_exception = ' {0:40} ==> '.format(img_filename_only)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs
else:
txt_exception += txt_organize_input + '\n'
return None, txt_exception
#select only reflections matched with scale input params.
#filter by resolution
i_sel_res = observations_original.resolution_filter_selection(
d_min=iparams.scale.d_min, d_max=iparams.scale.d_max)
observations_original_sel = observations_original.select(i_sel_res)
alpha_angle_sel = alpha_angle.select(i_sel_res)
spot_pred_x_mm_sel = spot_pred_x_mm.select(i_sel_res)
spot_pred_y_mm_sel = spot_pred_y_mm.select(i_sel_res)
#filter by sigma
i_sel_sigmas = (
observations_original_sel.data() /
observations_original_sel.sigmas()) > iparams.scale.sigma_min
observations_original_sel = observations_original_sel.select(
i_sel_sigmas)
alpha_angle_sel = alpha_angle_sel.select(i_sel_sigmas)
spot_pred_x_mm_sel = spot_pred_x_mm_sel.select(i_sel_sigmas)
spot_pred_y_mm_sel = spot_pred_y_mm_sel.select(i_sel_sigmas)
observations_non_polar_sel, index_basis_name = self.get_observations_non_polar(
observations_original_sel, pickle_filename, iparams)
observations_non_polar, index_basis_name = self.get_observations_non_polar(
observations_original, pickle_filename, iparams)
uc_params = observations_original.unit_cell().parameters()
ph = partiality_handler()
r0 = ph.calc_spot_radius(
sqr(crystal_init_orientation.reciprocal_matrix()),
observations_original_sel.indices(), wavelength)
#calculate first G
(G, B) = (1, 0)
stats = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
if mean_of_mean_I > 0:
G = flex.median(observations_original_sel.data()) / mean_of_mean_I
if iparams.flag_apply_b_by_frame:
try:
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(iparams.n_residues)
observations_as_f = observations_non_polar_sel.as_amplitude_array(
)
binner_template_asu = observations_as_f.setup_binner(
auto_binning=True)
wp = statistics.wilson_plot(observations_as_f,
asu_contents,
e_statistics=True)
G = wp.wilson_intensity_scale_factor * 1e2
B = wp.wilson_b
except Exception:
txt_exception += 'warning B-factor calculation failed.\n'
return None, txt_exception
two_theta = observations_original.two_theta(
wavelength=wavelength).data()
sin_theta_over_lambda_sq = observations_original.two_theta(
wavelength=wavelength).sin_theta_over_lambda_sq().data()
ry, rz, re, voigt_nu, rotx, roty = (0, 0, iparams.gamma_e,
iparams.voigt_nu, 0, 0)
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
crystal_init_orientation.unit_cell(),
rotx, roty, observations_original.indices(),
ry, rz, r0, re, voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm,
detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
if iparams.flag_plot_expert:
n_bins = 20
binner = observations_original.setup_binner(n_bins=n_bins)
binner_indices = binner.bin_indices()
avg_partiality_init = flex.double()
avg_rs_init = flex.double()
avg_rh_init = flex.double()
one_dsqr_bin = flex.double()
for i in range(1, n_bins + 1):
i_binner = (binner_indices == i)
if len(observations_original.data().select(i_binner)) > 0:
print binner.bin_d_range(i)[1], flex.mean(
partiality_init.select(i_binner)), flex.mean(
rs_init.select(i_binner)), flex.mean(
rh_init.select(i_binner)), len(
partiality_init.select(i_binner))
#monte-carlo merge
if iparams.flag_monte_carlo:
G = 1
B = 0
partiality_init = flex.double([1] * len(partiality_init))
#save results
refined_params = flex.double([
G, B, rotx, roty, ry, rz, r0, re, voigt_nu, uc_params[0],
uc_params[1], uc_params[2], uc_params[3], uc_params[4],
uc_params[5]
])
pres = postref_results()
pres.set_params(observations=observations_non_polar,
observations_original=observations_original,
refined_params=refined_params,
stats=stats,
partiality=partiality_init,
rs_set=rs_init,
rh_set=rh_init,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_init_orientation,
detector_distance_mm=detector_distance_mm)
txt_scale_frame_by_mean_I = ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} G:{3:6.4f} B:{4:6.1f} CELL:{5:6.2f} {6:6.2f} {7:6.2f} {8:6.2f} {9:6.2f} {10:6.2f}'.format(
img_filename_only + ' (' + index_basis_name + ')',
observations_original.d_min(),
len(observations_original_sel.data()), G, B, uc_params[0],
uc_params[1], uc_params[2], uc_params[3], uc_params[4],
uc_params[5])
print txt_scale_frame_by_mean_I
txt_scale_frame_by_mean_I += '\n'
return pres, txt_scale_frame_by_mean_I
def set_params(self,
observations=None,
observations_original=None,
G_lowres=None,
postref_params=None,
partiality=None,
rh_set=None,
rs_set=None,
stats=None,
bragg_angle_set=None,
alpha_angle_set=None,
spot_pred_x_mm_set=None,
spot_pred_y_mm_set=None,
wavelength=None,
crystal_orientation=None,
detector_distance_mm=None,
pickle_filename=None):
"""
reset observations and postrefinement parameters
"""
if observations is not None:
self.observations = observations.deep_copy()
if observations_original is not None:
self.observations_original = observations_original.deep_copy()
if G_lowres is not None:
self.G_lowres = G_lowres
if postref_params is not None:
self.postref_params = postref_params[:]
self.G, self.b11, self.b22, self.b33, self.b12, self.b13, self.b23, self.rotx, self.roty, self.ry, self.rz, self.r0, self.re = postref_params[:
13]
self.uc_params = flex.double(postref_params[13:])
self.unit_cell = unit_cell(
(self.uc_params[0], self.uc_params[1], self.uc_params[2],
self.uc_params[3], self.uc_params[4], self.uc_params[5]))
if partiality is not None:
self.partiality = partiality[:]
if rh_set is not None:
self.rh_set = rh_set[:]
if rs_set is not None:
self.rs_set = rs_set[:]
if stats is not None:
self.stats = stats[:]
self.cc_init = stats[0]
self.cc_final = stats[1]
self.r_init = stats[2]
self.r_final = stats[3]
self.r_xy_init = stats[4]
self.r_xy_final = stats[5]
self.cc_iso_init = stats[6]
self.cc_iso_final = stats[7]
if self.r_init == 0:
self.r_change = 0
else:
self.r_change = (
(self.r_final - self.r_init) / self.r_init) * 100
if self.cc_init == 0:
self.cc_change = 0
else:
self.cc_change = (
(self.cc_final - self.cc_init) / self.cc_init) * 100
if bragg_angle_set is not None:
self.bragg_angle_set = bragg_angle_set[:]
if alpha_angle_set is not None:
self.alpha_angle_set = alpha_angle_set[:]
if spot_pred_x_mm_set is not None:
self.spot_pred_x_mm_set = spot_pred_x_mm_set[:]
if spot_pred_y_mm_set is not None:
self.spot_pred_y_mm_set = spot_pred_y_mm_set[:]
if wavelength is not None:
self.wavelength = wavelength
if crystal_orientation is not None:
self.crystal_orientation = crystal_orientation
if detector_distance_mm is not None:
self.detector_distance_mm = detector_distance_mm
if pickle_filename is not None:
self.pickle_filename = pickle_filename
pickle_filepaths = pickle_filename.split('/')
if len(pickle_filepaths) > 0:
self.pickle_filename_only = pickle_filepaths[
len(pickle_filepaths) - 1]
else:
self.pickle_filename_only = pickle_filename
#set partiality
if self.postref_params is not None and self.partiality is None and self.crystal_orientation is not None:
partiality_hdlr = partiality_handler(self.iparams)
self.partiality, dum0, self.rh_set, self.rs_set = partiality_hdlr.calc_partiality_anisotropy_set(
self)
#get asu_contents
self.asu_contents = {}
asu_volume = self.observations.unit_cell().volume() / float(
self.observations.space_group().order_z())
number_carbons = asu_volume / 18.0
self.asu_contents.setdefault('C', number_carbons)
def determine_polar(self, observations_original, iparams, pickle_filename, pres=None):
"""
Determine polarity based on input data.
The function still needs isomorphous reference so, if flag_polar is True,
miller_array_iso must be supplied in input file.
"""
if iparams.indexing_ambiguity.flag_on == False:
return 'h,k,l', 0 , 0
cc_asu = 0
cc_rev = 0
if iparams.indexing_ambiguity.index_basis_in is not None:
if iparams.indexing_ambiguity.index_basis_in.endswith('mtz'):
#use reference mtz file to determine polarity
from iotbx import reflection_file_reader
reflection_file_polar = reflection_file_reader.any_reflection_file(iparams.indexing_ambiguity.index_basis_in)
miller_arrays_polar=reflection_file_polar.as_miller_arrays()
miller_array_polar = miller_arrays_polar[0]
miller_array_polar = miller_array_polar.resolution_filter(d_min=iparams.indexing_ambiguity.d_min, d_max=iparams.indexing_ambiguity.d_max)
#for post-refinement, apply the scale factors and partiality first
if pres is not None:
#observations_original = pres.observations_original.deep_copy()
two_theta = observations_original.two_theta(wavelength=pres.wavelength).data()
alpha_angle = flex.double([0]*len(observations_original.indices()))
spot_pred_x_mm = flex.double([0]*len(observations_original.indices()))
spot_pred_y_mm = flex.double([0]*len(observations_original.indices()))
detector_distance_mm = pres.detector_distance_mm
ph = partiality_handler()
partiality, dummy, dummy, dummy = ph.calc_partiality_anisotropy_set(pres.unit_cell, 0, 0,
observations_original.indices(),
pres.ry, pres.rz, pres.r0, pres.re,
two_theta, alpha_angle, pres.wavelength, pres.crystal_orientation,
spot_pred_x_mm, spot_pred_y_mm,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
#partiality = pres.partiality
sin_theta_over_lambda_sq = observations_original.two_theta(pres.wavelength).sin_theta_over_lambda_sq().data()
I_full = flex.double(observations_original.data()/(pres.G * flex.exp(flex.double(-2*pres.B*sin_theta_over_lambda_sq)) * partiality))
sigI_full = flex.double(observations_original.sigmas()/(pres.G * flex.exp(flex.double(-2*pres.B*sin_theta_over_lambda_sq)) * partiality))
observations_original = observations_original.customized_copy(data=I_full, sigmas=sigI_full)
observations_asu = observations_original.map_to_asu()
observations_rev = self.get_observations_non_polar(observations_original, iparams.indexing_ambiguity.assigned_basis)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_polar.indices(),
miller_indices=observations_asu.indices())
I_ref_match = flex.double([miller_array_polar.data()[pair[0]] for pair in matches.pairs()])
I_obs_match = flex.double([observations_asu.data()[pair[1]] for pair in matches.pairs()])
cc_asu = flex.linear_correlation(I_ref_match, I_obs_match).coefficient()
n_refl_asu = len(matches.pairs())
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_polar.indices(),
miller_indices=observations_rev.indices())
I_ref_match = flex.double([miller_array_polar.data()[pair[0]] for pair in matches.pairs()])
I_obs_match = flex.double([observations_rev.data()[pair[1]] for pair in matches.pairs()])
cc_rev = flex.linear_correlation(I_ref_match, I_obs_match).coefficient()
n_refl_rev = len(matches.pairs())
polar_hkl = 'h,k,l'
if cc_rev > (cc_asu*1.01):
polar_hkl = iparams.indexing_ambiguity.assigned_basis
else:
#use basis in the given input file
polar_hkl = 'h,k,l'
basis_pickle = pickle.load(open(iparams.indexing_ambiguity.index_basis_in,"rb"))
if pickle_filename in basis_pickle:
polar_hkl = basis_pickle[pickle_filename]
else:
#set default polar_hkl to h,k,l
polar_hkl = 'h,k,l'
return polar_hkl, cc_asu, cc_rev
def func(self, params, args):
I_r = args[0]
miller_array_o = args[1]
wavelength = args[2]
alpha_angle_set = args[3]
crystal_init_orientation = args[4]
spot_pred_x_mm_set = args[5]
spot_pred_y_mm_set = args[6]
detector_distance_mm = args[7]
refine_mode = args[8]
const_params = args[9]
b0 = args[10]
miller_array_iso = args[11]
iparams = args[12]
partiality_model = iparams.partiality_model
flag_volume_correction = iparams.flag_volume_correction
flag_beam_divergence = iparams.flag_beam_divergence
b_refine_d_min = iparams.b_refine_d_min
I_o = miller_array_o.data()
sigI_o = miller_array_o.sigmas()
miller_indices_original = miller_array_o.indices()
sin_theta_over_lambda_sq = miller_array_o.two_theta(
wavelength=wavelength).sin_theta_over_lambda_sq().data()
two_theta_flex = miller_array_o.two_theta(wavelength=wavelength).data()
cs = miller_array_o.crystal_symmetry().space_group().crystal_system()
if refine_mode == 'scale_factor':
G, B = params
rotx, roty, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = const_params
elif refine_mode == 'crystal_orientation':
rotx, roty = params
G, B, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = const_params
elif refine_mode == 'reflecting_range':
ry, rz, r0, re, nu = params
G, B, rotx, roty, a, b, c, alpha, beta, gamma = const_params
elif refine_mode == 'unit_cell':
a, b, c, alpha, beta, gamma = self.prep_output(params, cs)
G, B, rotx, roty, ry, rz, r0, re, nu = const_params
elif refine_mode == 'allparams':
a, b, c, alpha, beta, gamma = self.prep_output(params[7:], cs)
rotx, roty, ry, rz, r0, re, nu = params[0:7]
G, B = const_params
try:
uc = unit_cell((a, b, c, alpha, beta, gamma))
except Exception:
return None
G, B, rotx, roty, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = flex.double(
[
G, B, rotx, roty, ry, rz, r0, re, nu, a, b, c, alpha, beta,
gamma
])
ph = partiality_handler()
p_calc_flex, delta_xy_flex, rs_set, dummy = ph.calc_partiality_anisotropy_set(\
uc, rotx, roty, miller_indices_original, ry, rz, r0, re, nu, two_theta_flex,
alpha_angle_set, wavelength, crystal_init_orientation, spot_pred_x_mm_set,
spot_pred_y_mm_set, detector_distance_mm, partiality_model, flag_beam_divergence)
if miller_array_o.d_min() < b_refine_d_min:
I_o_full = ph.calc_full_refl(I_o, sin_theta_over_lambda_sq, G, B,
p_calc_flex, rs_set,
flag_volume_correction)
else:
I_o_full = ph.calc_full_refl(I_o, sin_theta_over_lambda_sq, G, b0,
p_calc_flex, rs_set,
flag_volume_correction)
if refine_mode == 'unit_cell':
error = delta_xy_flex
else:
error = ((I_r - I_o_full) / sigI_o)
"""
import math
print refine_mode, 'G:%.4g B:%.4g rotx:%.4g roty:%.4g r0:%.4g re:%.4g nu:%6.4f a:%.4g b:%.4g c:%.4g fpr:%.4g fxy:%.4g n_refl=%5.0f'%(G, B, rotx*180/math.pi, roty*180/math.pi, r0, re, nu, a, b, c, flex.sum(((I_r - I_o_full)/sigI_o)**2), flex.sum(delta_xy_flex**2), len(I_o_full))
"""
return error
def postrefine_by_frame(self, frame_no, pres_in, iparams, miller_array_ref, avg_mode):
#Prepare data
if pres_in is None:
return None, 'Found empty pickle file'
observations_pickle = pickle.load(open(pres_in.pickle_filename,"rb"))
wavelength = observations_pickle["wavelength"]
crystal_init_orientation = observations_pickle["current_orientation"][0]
pickle_filename = pres_in.pickle_filename
pickle_filepaths = pickle_filename.split('/')
img_filename_only = pickle_filepaths[len(pickle_filepaths)-1]
txt_exception = ' {0:40} ==> '.format(img_filename_only)
inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
else:
txt_exception += txt_organize_input + '\n'
return None, txt_exception
#Select data for post-refinement (only select indices that are common with the reference set
observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_ref.indices(),
miller_indices=observations_non_polar.indices())
I_ref_match = flex.double([miller_array_ref.data()[pair[0]] for pair in matches.pairs()])
miller_indices_ref_match = flex.miller_index((miller_array_ref.indices()[pair[0]] for pair in matches.pairs()))
I_obs_match = flex.double([observations_non_polar.data()[pair[1]] for pair in matches.pairs()])
sigI_obs_match = flex.double([observations_non_polar.sigmas()[pair[1]] for pair in matches.pairs()])
miller_indices_original_obs_match = flex.miller_index((observations_original.indices()[pair[1]] \
for pair in matches.pairs()))
miller_indices_non_polar_obs_match = flex.miller_index((observations_non_polar.indices()[pair[1]] \
for pair in matches.pairs()))
alpha_angle_set = flex.double([alpha_angle[pair[1]] for pair in matches.pairs()])
spot_pred_x_mm_set = flex.double([spot_pred_x_mm[pair[1]] for pair in matches.pairs()])
spot_pred_y_mm_set = flex.double([spot_pred_y_mm[pair[1]] for pair in matches.pairs()])
references_sel = miller_array_ref.customized_copy(data=I_ref_match, indices=miller_indices_ref_match)
observations_original_sel = observations_original.customized_copy(data=I_obs_match,
sigmas=sigI_obs_match,
indices=miller_indices_original_obs_match)
observations_non_polar_sel = observations_non_polar.customized_copy(data=I_obs_match,
sigmas=sigI_obs_match,
indices=miller_indices_non_polar_obs_match)
#Do least-squares refinement
lsqrh = leastsqr_handler()
try:
refined_params, stats, n_refl_postrefined = lsqrh.optimize(I_ref_match,
observations_original_sel, wavelength,
crystal_init_orientation, alpha_angle_set,
spot_pred_x_mm_set, spot_pred_y_mm_set,
iparams,
pres_in,
observations_non_polar_sel,
detector_distance_mm)
except Exception:
txt_exception += 'optimization failed.\n'
return None, txt_exception
#caculate partiality for output (with target_anomalous check)
G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \
a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params
inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename)
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams)
from cctbx.uctbx import unit_cell
uc_fin = unit_cell((a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin))
crystal_init_orientation = pres_in.crystal_orientation
two_theta = observations_original.two_theta(wavelength=wavelength).data()
ph = partiality_handler()
partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(uc_fin, rotx_fin, roty_fin,
observations_original.indices(),
ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,
spot_pred_x_mm, spot_pred_y_mm,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
#calculate the new crystal orientation
O = sqr(uc_fin.orthogonalization_matrix()).transpose()
R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose()
from cctbx.crystal_orientation import crystal_orientation, basis_type
CO = crystal_orientation(O*R, basis_type.direct)
crystal_fin_orientation = CO.rotate_thru((1,0,0), rotx_fin
).rotate_thru((0,1,0), roty_fin)
#remove reflections with partiality below threshold
i_sel = partiality_fin > iparams.merge.partiality_min
partiality_fin_sel = partiality_fin.select(i_sel)
rs_fin_sel = rs_fin.select(i_sel)
rh_fin_sel = rh_fin.select(i_sel)
observations_non_polar_sel = observations_non_polar.select(i_sel)
observations_original_sel = observations_original.select(i_sel)
mapped_predictions = mapped_predictions.select(i_sel)
pres = postref_results()
pres.set_params(observations = observations_non_polar_sel,
observations_original = observations_original_sel,
refined_params=refined_params,
stats=stats,
partiality=partiality_fin_sel,
rs_set=rs_fin_sel,
rh_set=rh_fin_sel,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_init_orientation,
detector_distance_mm=detector_distance_mm,
identified_isoform=identified_isoform,
mapped_predictions=mapped_predictions,
xbeam=xbeam,
ybeam=ybeam)
r_change, r_xy_change, cc_change, cc_iso_change = (0,0,0,0)
try:
r_change = ((pres.R_final - pres.R_init)/pres.R_init)*100
r_xy_change = ((pres.R_xy_final - pres.R_xy_init)/pres.R_xy_init)*100
cc_change = ((pres.CC_final - pres.CC_init)/pres.CC_init)*100
cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init)/pres.CC_iso_init)*100
except Exception:
pass
txt_postref= ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}'.format(img_filename_only+' ('+index_basis_name+')', observations_original_sel.d_min(), len(observations_original_sel.data()), r_change, r_xy_change, cc_change, cc_iso_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin)
print txt_postref
txt_postref += '\n'
return pres, txt_postref
def jacobian_callable(pfh,current_values):
rotx = current_values[0]
roty = current_values[1]
ry = spot_radius
rz = spot_radius
G = scale_factors[0]
B_factor = scale_factors[1]
I_obs = observations_original_sel.data()
sigI_obs = observations_original_sel.sigmas()
observations_original_sel_two_theta = observations_original_sel.two_theta(wavelength=wavelength)
two_theta = observations_original_sel_two_theta.data()
sin_theta_over_lambda_sq = observations_original_sel_two_theta.sin_theta_over_lambda_sq().data()
delta_rot = 0.0001*math.pi/180
delta_r = 0.00000001
#0. Calculate current partiality based on current rotx
crystal_current_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx
).rotate_thru((0,1,0),roty)
a_star = sqr(crystal_current_orientation.reciprocal_matrix())
ph = partiality_handler(wavelength, 0)
#1. Calculate partial derivatives of J function
Ai = sqr(crystal_init_orientation.reciprocal_matrix())
Rx = col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(rotx)
Ry = col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(roty)
Rz = col((0,0,1)).axis_and_angle_as_r3_rotation_matrix(0.0)
dRx_drotx = col((1,0,0)).axis_and_angle_as_r3_derivative_wrt_angle(rotx)
dRy_droty = col((0,1,0)).axis_and_angle_as_r3_derivative_wrt_angle(roty)
dA_drotx = Rz * Ry * dRx_drotx * Ai
dA_droty = Rz * dRy_droty * Rx * Ai
s0 = -1*col((0,0,1./wavelength))
pDg_pDrotx = flex.double()
pDg_pDroty = flex.double()
pDg_pDry = flex.double()
pDg_pDrz = flex.double()
for j in range(len(observations_original_sel.indices())):
miller_index = observations_original_sel.indices()[j]
hvec = col(miller_index)
xvec = a_star * hvec
svec = s0 + xvec
p, rh, rs = ph.calc_partiality_anisotropy(a_star, miller_index, ry, rz, two_theta[j])
#for rotx
pDxvec_pDrotx = dA_drotx * hvec
pDrh_pDrotx = (svec.dot(pDxvec_pDrotx))/svec.length()
#for roty
pDxvec_pDroty = dA_droty * hvec
pDrh_pDroty = (svec.dot(pDxvec_pDroty))/svec.length()
#shared derivatives on rotx and roty
pDP_pDrh = (-4*(rs**2)*rh)/((2*(rh**2))+(rs**2))**2
pDg_pDP = (-1 * G * (math.exp(-2*B_factor*sin_theta_over_lambda_sq[j])) * I_obs[j])/(sigI_obs[j]*(p**2))
pDg_pDrotx.append(pDg_pDP * pDP_pDrh * pDrh_pDrotx)
pDg_pDroty.append(pDg_pDP * pDP_pDrh * pDrh_pDroty)
"""
#checking partial derivatives
g_now = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx,roty,ry,rz))
g_delta_rotx = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx+delta_rot,roty,ry,rz))
g_delta_roty = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx,roty+delta_rot,ry,rz))
pDg_pDrotx_fd = (g_delta_rotx - g_now)/delta_rot
pDg_pDroty_fd = (g_delta_roty - g_now)/delta_rot
print sum(flex.abs(pDg_pDrotx_fd)), sum(flex.abs(pDg_pDrotx_fd-pDg_pDrotx))
print sum(flex.abs(pDg_pDroty_fd)), sum(flex.abs(pDg_pDroty_fd-pDg_pDroty))
print
"""
return pDg_pDrotx, pDg_pDroty
def gradient_descend(self,I_ref, I_obs, sigI_obs, crystal_init_orientation,
miller_indices_ori, wavelength, parameters, alpha, n_iters):
m = len(I_ref)
G = parameters[0]
rotx = parameters[1]
roty = parameters[2]
rotz = parameters[3]
rs = parameters[4]
delta_rot = 0.00001*math.pi/180
J_history = flex.double([0]*n_iters)
converge_iter = n_iters - 1
for i in range(n_iters):
#0. Calculate current partiality based on current rotation
crystal_current_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx
).rotate_thru((0,1,0),roty
).rotate_thru((0,0,1),rotz)
a_star = sqr(crystal_current_orientation.reciprocal_matrix())
ph = partiality_handler(wavelength, rs)
partiality = flex.double([ph.calc_partiality(a_star, miller_index)[0] for miller_index in miller_indices_ori])
#1. Calculate partial derivatives of J function
Ai = sqr(crystal_init_orientation.reciprocal_matrix())
Rx = col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(rotx)
Ry = col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(roty)
Rz = col((0,0,1)).axis_and_angle_as_r3_rotation_matrix(rotz)
dRx_drotx = col((1,0,0)).axis_and_angle_as_r3_derivative_wrt_angle(rotx)
dRy_droty = col((0,1,0)).axis_and_angle_as_r3_derivative_wrt_angle(roty)
dRz_drotz = col((0,0,1)).axis_and_angle_as_r3_derivative_wrt_angle(rotz)
dA_drotx = Rz * Ry * dRx_drotx * Ai
dA_droty = Rz * dRy_droty * Rx * Ai
dA_drotz = dRz_drotz * Ry * Rx * Ai
s0 = -1*col((0,0,1./wavelength))
pDJ_pDrotx = 0
pDJ_pDroty = 0
pDJ_pDrotz = 0
pDJ_pDG = 0
pDJ_pDrs = 0
for j in range(len(miller_indices_ori)):
miller_index = miller_indices_ori[j]
hvec = col(miller_index)
xvec = a_star * hvec
svec = s0 + xvec
p, rh = ph.calc_partiality(a_star, miller_index)
#for rotx
pDxvec_pDrotx = dA_drotx * hvec
pDrh_pDrotx = (svec.dot(pDxvec_pDrotx))/svec.length()
"""
#for finite differences
#Rx_deltax = col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(rotx+delta_rot)
#a_star_deltax = Ry * Rx_deltax * Ai
crystal_deltax_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx+delta_rot
).rotate_thru((0,1,0),roty
).rotate_thru((0,0,1),rotz)
a_star_deltax = sqr(crystal_deltax_orientation.reciprocal_matrix())
p_deltax, rh_deltax = ph.calc_partiality(a_star_deltax, miller_index)
xvec_deltax = a_star_deltax * hvec
svec_deltax = s0 + xvec_deltax
pDxvec_pDrotx_fd = (xvec_deltax - xvec)/delta_rot
pDrh_pDrotx_fd = (rh_deltax - rh)/delta_rot
if j==1:
print pDrh_pDrotx, pDrh_pDrotx_fd, pDrh_pDrotx-pDrh_pDrotx_fd
"""
#for roty
pDxvec_pDroty = dA_droty * hvec
pDrh_pDroty = (svec.dot(pDxvec_pDroty))/svec.length()
"""
#for finite differences
crystal_deltay_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx
).rotate_thru((0,1,0),roty+delta_rot
).rotate_thru((0,0,1),rotz)
a_star_deltay = sqr(crystal_deltay_orientation.reciprocal_matrix())
p_deltay, rh_deltay = ph.calc_partiality(a_star_deltay, miller_index)
xvec_deltay = a_star_deltay * hvec
svec_deltay = s0 + xvec_deltay
pDxvec_pDroty_fd = (xvec_deltay - xvec)/delta_rot
pDrh_pDroty_fd = (rh_deltay - rh)/delta_rot
if j==1:
print pDrh_pDroty, pDrh_pDroty_fd, pDrh_pDroty-pDrh_pDroty_fd
"""
#for rotz
pDxvec_pDrotz = dA_drotz * hvec
pDrh_pDrotz = (svec.dot(pDxvec_pDrotz))/svec.length()
#for finite differences
crystal_deltaz_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx
).rotate_thru((0,1,0),roty
).rotate_thru((0,0,1),rotz+delta_rot)
a_star_deltaz = sqr(crystal_deltaz_orientation.reciprocal_matrix())
p_deltaz, rh_deltaz = ph.calc_partiality(a_star_deltaz, miller_index)
xvec_deltaz = a_star_deltaz * hvec
svec_deltaz = s0 + xvec_deltaz
pDxvec_pDrotz = (xvec_deltaz - xvec)/delta_rot
pDrh_pDrotz = (rh_deltaz - rh)/delta_rot
#shared derivatives on rotx and roty
pDP_pDrh = (-4*(rs**2)*rh)/((2*(rh**2))+(rs**2))**2
pDg_pDP = (-1 * G * I_obs[j])/(sigI_obs[j]*(p**2))
#for G
pDg_pDG = I_obs[j] / (sigI_obs[j] * p)
#for rs
pDP_pDrs = (4*rs*(rh**2))/(((2*(rh**2))+(rs**2))**2)
pDJ = ((((G * I_obs[j])/p) - I_ref[j])/sigI_obs[j])/m
pDJ_pDrotx += pDJ * pDg_pDP * pDP_pDrh * pDrh_pDrotx
pDJ_pDroty += pDJ * pDg_pDP * pDP_pDrh * pDrh_pDroty
pDJ_pDrotz += pDJ * pDg_pDP * pDP_pDrh * pDrh_pDrotz
pDJ_pDG += pDJ * pDg_pDG
pDJ_pDrs += pDJ * pDg_pDP * pDP_pDrs
"""
#finite differences for PDJ
#Calculate finite differences of J function over rotation angle x and y
J_at_rotx_and_delta = compute_cost(I_ref, I_obs, crystal_init_orientation, miller_indices, ph, (k, G, rotx+delta_rot, roty))
J_at_roty_and_delta = compute_cost(I_ref, I_obs, crystal_init_orientation, miller_indices, ph, (k, G, rotx, roty+delta_rot))
J_at_rot = compute_cost(I_ref, I_obs, crystal_init_orientation, miller_indices, ph, (k, G, rotx, roty))
pDJ_pDrotx_fd = (J_at_rotx_and_delta - J_at_rot)/delta_rot
pDJ_pDroty_fd = (J_at_roty_and_delta - J_at_rot)/delta_rot
print i, pDJ_pDrotx, pDJ_pDrotx_fd, pDJ_pDroty, pDJ_pDroty_fd
"""
#update parameters
rotx = rotx - (alpha*pDJ_pDrotx)
roty = roty - (alpha*pDJ_pDroty)
rotz = rotz - (alpha*pDJ_pDrotz)
G = G - (alpha*pDJ_pDG)
rs = rs - (alpha*pDJ_pDrs)
J_history[i] = self.compute_cost(I_ref, I_obs, sigI_obs, crystal_init_orientation, miller_indices_ori, wavelength, (G, rotx, roty, rotz, rs))
print i, G, rotx*180/math.pi, roty*180/math.pi, rotz*180/math.pi, rs
#check convergence
gradient_threshold = 0.05
if i > 0:
if abs((J_history[i]-J_history[i-1])/delta_rot) < gradient_threshold:
converge_iter = i
break
return G, rotx, roty, rotz, rs
def optimize(self, I_r_flex, observations_original, wavelength,
crystal_init_orientation, alpha_angle, spot_pred_x_mm,
spot_pred_y_mm, iparams, pres_in, observations_non_polar,
detector_distance_mm):
ph = partiality_handler()
lph = lbfgs_partiality_handler()
if iparams.postref.allparams.flag_on:
refine_steps = ['allparams']
else:
refine_steps = ['crystal_orientation']
if iparams.postref.reflecting_range.flag_on:
refine_steps.append('reflecting_range')
if iparams.postref.unit_cell.flag_on:
refine_steps.append('unit_cell')
#get miller array iso, if given.
miller_array_iso = None
#prepare data
pr_d_min = iparams.postref.allparams.d_min
pr_d_max = iparams.postref.allparams.d_max
pr_sigma_min = iparams.postref.allparams.sigma_min
pr_partiality_min = iparams.postref.allparams.partiality_min
pr_uc_tol = iparams.postref.allparams.uc_tolerance
cs = observations_original.crystal_symmetry().space_group(
).crystal_system()
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
#initialize values only in the first sub cycle and the first refine step.
spot_radius = ph.calc_spot_radius(
sqr(crystal_init_orientation.reciprocal_matrix()),
observations_original_sel.indices(), wavelength)
if pres_in is None:
ry, rz, r0, re, voigt_nu, rotx, roty = 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, 0.0, 0.0
#apply constrain on the unit cell using crystal system
uc_scale_inp = lph.prep_input(
observations_original.unit_cell().parameters(), cs)
uc_scale_constrained = lph.prep_output(uc_scale_inp, cs)
a, b, c, alpha, beta, gamma = uc_scale_constrained
const_params_scale = (rotx, roty, ry, rz, r0, re, voigt_nu, a, b,
c, alpha, beta, gamma)
xopt_scalefactors, stats = self.optimize_scalefactors(
I_r_flex, observations_original, wavelength,
crystal_init_orientation, alpha_angle, spot_pred_x_mm,
spot_pred_y_mm, iparams, pres_in, observations_non_polar,
detector_distance_mm, const_params_scale)
G, B = xopt_scalefactors
else:
G, B, ry, rz, r0, re, voigt_nu, rotx, roty = pres_in.G, pres_in.B, pres_in.ry, pres_in.rz, pres_in.r0, pres_in.re, pres_in.voigt_nu, 0.0, 0.0
a, b, c, alpha, beta, gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#filter by partiality
two_theta = observations_original_sel.two_theta(
wavelength=wavelength).data()
uc = unit_cell((a, b, c, alpha, beta, gamma))
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(
uc, rotx, roty, observations_original_sel.indices(), ry, rz, r0,
re, voigt_nu, two_theta, alpha_angle_sel, wavelength,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'partiality', [pr_partiality_min], observations_original_sel,
alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel,
partiality_in=partiality_init)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()[:]
#calculate initial residual_xy error
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell', const_params_uc, B,
miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
init_residual_xy_err = flex.sum(uc_params_err**2)
#calculate initial residual_pr error
const_params_all = (G, B)
xinp_all = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp_all.extend(lph.prep_input((a, b, c, alpha, beta, gamma), cs))
args_all = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'allparams', const_params_all, B,
miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
init_residual_err = flex.sum(all_params_err**2)
#keep in list
t_pr_list = [init_residual_err]
t_xy_list = [init_residual_xy_err]
refined_params_hist = [(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,
b, c, alpha, beta, gamma)]
txt_out = ''
for i_sub_cycle in range(iparams.n_postref_sub_cycle):
for j_refine_step in range(len(refine_steps)):
refine_mode = refine_steps[j_refine_step]
#prepare data
init_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,
b, c, alpha, beta, gamma)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.prepare_data_microcycle(refine_mode, iparams,
observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm,
I_r_flex, init_params, crystal_init_orientation,
wavelength, detector_distance_mm)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()
if refine_mode == 'crystal_orientation':
xinp = flex.double([rotx, roty])
const_params = (G, B, ry, rz, r0, re, voigt_nu, a, b, c,
alpha, beta, gamma)
elif refine_mode == 'reflecting_range':
xinp = flex.double([ry, rz, r0, re, voigt_nu])
const_params = (G, B, rotx, roty, a, b, c, alpha, beta,
gamma)
elif refine_mode == 'unit_cell':
xinp = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
const_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
elif refine_mode == 'allparams':
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(
lph.prep_input((a, b, c, alpha, beta, gamma), cs))
const_params = (G, B)
args = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, refine_mode, const_params, B,
miller_array_iso, iparams)
lh = lbfgs_handler(current_x=xinp, args=args)
xopt = flex.double(list(lh.x))
if refine_mode == 'crystal_orientation' or \
refine_mode == 'reflecting_range' or refine_mode == 'allparams':
current_residual_err = lh.f
#calculate residual_xy_error (for refine_mode = SF, CO, RR, and all params)
xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
if refine_mode == 'crystal_orientation':
rotx, roty = xopt
elif refine_mode == 'reflecting_range':
ry, rz, r0, re, voigt_nu = xopt
elif refine_mode == 'allparams':
rotx, roty, ry, rz, r0, re, voigt_nu = xopt[:7]
xinp_uc = xopt[7:]
a, b, c, alpha, beta, gamma = lph.prep_output(
xinp_uc, cs)
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re,
voigt_nu)
xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell',
const_params_uc, B, miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
current_residual_xy_err = flex.sum(uc_params_err**2)
elif refine_mode == 'unit_cell':
current_residual_xy_err = lh.f
xopt_uc = lph.prep_output(xopt, cs)
a, b, c, alpha, beta, gamma = xopt_uc
#check the unit-cell with the reference intensity
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(
lph.prep_input((a, b, c, alpha, beta, gamma), cs))
const_params_all = (G, B)
args_all = (I_r_true, observations_original_sel,
wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel,
spot_pred_y_mm_sel, detector_distance_mm,
'allparams', const_params_all, B,
miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
current_residual_err = flex.sum(all_params_err**2)
flag_success = False
if refine_mode == 'allparams':
#if allparams refinement, only check the post-refine target function
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append(
(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b,
c, alpha, beta, gamma))
flag_success = True
else:
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
if current_residual_xy_err < (t_xy_list[len(t_xy_list)-1] + \
(t_xy_list[len(t_xy_list)-1]*iparams.postref.residual_threshold_xy/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append(
(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,
b, c, alpha, beta, gamma))
flag_success = True
if flag_success is False:
G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = refined_params_hist[
len(refined_params_hist) - 1]
tmp_txt_out = refine_mode + ' %3.0f %6.4f %6.4f %6.4f %6.4f %10.8f %10.8f %10.8f %10.8f %10.8f %6.3f %6.3f %.4g %6.3f\n' % (
i_sub_cycle, G, B, rotx * 180 / math.pi, roty * 180 /
math.pi, ry, rz, r0, re, voigt_nu, a, c, t_pr_list[
len(t_pr_list) - 1], t_xy_list[len(t_pr_list) - 1])
txt_out += tmp_txt_out
#apply the refined parameters on the full (original) reflection set
two_theta = observations_original.two_theta(
wavelength=wavelength).data()
sin_theta_over_lambda_sq = observations_original.two_theta(
wavelength=wavelength).sin_theta_over_lambda_sq().data()
if pres_in is None:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
observations_original.unit_cell(),0.0, 0.0,observations_original.indices(),
0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, 1, 0,
partiality_init, rs_init)
else:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
pres_in.unit_cell,0.0, 0.0,observations_original.indices(),
pres_in.ry, pres_in.rz,pres_in.r0, pres_in.re, pres_in.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, pres_in.G,
pres_in.B, partiality_init, rs_init)
partiality_fin, delta_xy_fin, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(\
unit_cell((a,b,c,alpha,beta,gamma)),rotx, roty,observations_original.indices(),
ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength,crystal_init_orientation,
spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_fin = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, G, B,
partiality_fin, rs_fin)
SE_of_the_estimate = standard_error_of_the_estimate(
I_r_flex, I_o_fin, 13)
R_sq = coefficient_of_determination(I_r_flex, I_o_fin) * 100
CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient()
CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient()
err_init = (I_r_flex - I_o_init) / observations_original.sigmas()
R_init = math.sqrt(flex.sum(err_init**2))
err_final = (I_r_flex - I_o_fin) / observations_original.sigmas()
R_final = math.sqrt(flex.sum(err_final**2))
R_xy_init = math.sqrt(flex.sum(delta_xy_init**2))
R_xy_final = math.sqrt(flex.sum(delta_xy_fin**2))
if R_init < R_final or re > (iparams.gamma_e * 3):
CC_final = CC_init
R_final = R_init
R_xy_final = R_xy_init
if pres_in is None:
G, B, r0, ry, rz, re, rotx, roty = (1.0, 0.0, spot_radius, 0.0,
0.0, iparams.gamma_e, 0.0,
0.0)
a, b, c, alpha, beta, gamma = observations_original.unit_cell(
).parameters()
else:
G, B, r0, ry, rz, re, rotx, roty = (pres_in.G, pres_in.B,
pres_in.r0, pres_in.ry,
pres_in.rz, pres_in.re,
0.0, 0.0)
a, b, c, alpha, beta, gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#calculate CCiso if hklisoin is given
CC_iso_init, CC_iso_final = (0, 0)
if iparams.hklisoin is not None:
if miller_array_iso is not None:
from cctbx import miller
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_iso.indices(),
miller_indices=observations_non_polar.indices())
I_iso_match = flex.double([
miller_array_iso.data()[pair[0]]
for pair in matches.pairs()
])
I_o_init_match = flex.double(
[I_o_init[pair[1]] for pair in matches.pairs()])
I_o_fin_match = flex.double(
[I_o_fin[pair[1]] for pair in matches.pairs()])
CC_iso_init = flex.linear_correlation(
I_iso_match, I_o_init_match).coefficient()
CC_iso_final = flex.linear_correlation(
I_iso_match, I_o_fin_match).coefficient()
xopt = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha,
beta, gamma)
return xopt, (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init,
R_final, R_xy_init, R_xy_final, CC_iso_init,
CC_iso_final), len(I_ref_sel)
def optimize_scalefactors(self, I_r_flex, observations_original,
wavelength, crystal_init_orientation,
alpha_angle, spot_pred_x_mm, spot_pred_y_mm,
iparams, pres_in, observations_non_polar,
detector_distance_mm, const_params):
ph = partiality_handler()
pr_d_min = iparams.postref.scale.d_min
pr_d_max = iparams.postref.scale.d_max
pr_sigma_min = iparams.postref.scale.sigma_min
pr_partiality_min = iparams.postref.scale.partiality_min
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,\
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,\
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()[:]
if pres_in is not None:
G, B, b0 = pres_in.G, pres_in.B, pres_in.B
else:
G, B, b0 = (1, 0, 0)
refine_mode = 'scale_factor'
xinp = flex.double([G, B])
args = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel,
spot_pred_y_mm_sel, detector_distance_mm, refine_mode,
const_params, b0, None, iparams)
lh = lbfgs_handler(current_x=xinp, args=args)
G_fin, B_fin = (lh.x[0], lh.x[1])
rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = const_params
two_theta = observations_original.two_theta(wavelength=wavelength)
sin_theta_over_lambda_sq = two_theta.sin_theta_over_lambda_sq().data()
uc = unit_cell((a, b, c, alpha, beta, gamma))
ph = partiality_handler()
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(
uc, rotx, roty, observations_original.indices(), ry, rz, r0, re,
voigt_nu, two_theta.data(), alpha_angle, wavelength,
crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm,
detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, G, B,
partiality_init, rs_init)
I_o_fin = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, G_fin, B_fin,
partiality_init, rs_init)
SE_of_the_estimate = standard_error_of_the_estimate(
I_r_flex, I_o_fin, 2)
R_sq = coefficient_of_determination(I_r_flex, I_o_fin) * 100
CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient()
CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient()
err_init = (I_r_flex - I_o_init) / observations_original.sigmas()
R_init = math.sqrt(flex.sum(err_init**2))
err_final = (I_r_flex - I_o_fin) / observations_original.sigmas()
R_final = math.sqrt(flex.sum(err_final**2))
R_xy_init = 0
R_xy_final = 0
CC_iso_init = 0
CC_iso_final = 0
return flex.double(list(lh.x)), (SE_of_the_estimate, R_sq, CC_init,
CC_final, R_init, R_final, R_xy_init,
R_xy_final, CC_iso_init, CC_iso_final)
def optimize(self, I_r_flex, observations_original,
wavelength, crystal_init_orientation, alpha_angle_set, iph):
assert len(alpha_angle_set)==len(observations_original.indices()), 'Size of alpha angles and observations are not equal %6.0f, %6.0f'%(len(alpha_angle_set),len(observations_original.indices()))
uc_init = crystal_init_orientation.unit_cell()
uc_init_params = uc_init.parameters()
I_r_true = I_r_flex.as_numpy_array()
I_o_true = observations_original.data().as_numpy_array()
sigI_o_true = observations_original.sigmas().as_numpy_array()
sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data().as_numpy_array()
two_theta_flex = observations_original.two_theta(wavelength=wavelength).data()
cs = observations_original.crystal_symmetry().space_group().crystal_system()
#calculate spot_radius
a_star_true = sqr(crystal_init_orientation.reciprocal_matrix())
spot_radius = calc_spot_radius(a_star_true, observations_original.indices(), wavelength)
ph = partiality_handler(wavelength, spot_radius)
#1. first optain best G and k from linregress
x0 = np.array([1, 0])
xopt_scale, success = optimize.leastsq(func_scale, x0, args=(I_r_true, observations_original, wavelength))
#2. optimize rot, rs, uc
x0_all = np.array([xopt_scale[0], xopt_scale[1], 0, 0, spot_radius, spot_radius, 0.0026,
uc_init_params[0], uc_init_params[1],uc_init_params[2], uc_init_params[3],uc_init_params[4], uc_init_params[5]])
x0 = prep_input(x0_all, cs)
try:
xopt_limit, cov_x, infodict, errmsg, success = optimize.leastsq(func, x0, args=(I_r_true, observations_original, ph, crystal_init_orientation, alpha_angle_set, iph), full_output=True)
except Exception:
xopt_limit = x0
cov_x = None
xopt = prep_output(xopt_limit, cs)
G, B, rotx, roty, ry, rz, re, a, b, c, alpha, beta, gamma = xopt
#3. decide wheter to take the refined parameters
if (abs(a-iph.target_unit_cell[0]) > iph.uc_len_tol or abs(b-iph.target_unit_cell[1]) > iph.uc_len_tol or abs(c-iph.target_unit_cell[2]) > iph.uc_len_tol \
or abs(alpha-iph.target_unit_cell[3]) > iph.uc_angle_tol or abs(beta-iph.target_unit_cell[4]) > iph.uc_angle_tol or abs(gamma-iph.target_unit_cell[5]) > iph.uc_angle_tol):
if iph.flag_force_accept_all_frames:
a, b, c, alpha, beta, gamma = uc_init_params
rotx = 0
roty = 0
ry = spot_radius
rz = spot_radius
re = 0.0026
G, B = xopt_scale
xopt = np.array([G, B, rotx, roty, ry, rz, re, a, b, c, alpha, beta, gamma])
#caclculate stats
uc_opt = unit_cell((a,b,c,alpha,beta,gamma))
crystal_orientation_opt = get_crystal_orientation(uc_opt.orthogonalization_matrix(), crystal_init_orientation.crystal_rotation_matrix())
crystal_orientation_model = crystal_orientation_opt.rotate_thru((1,0,0), rotx
).rotate_thru((0,1,0), roty)
a_star_model = sqr(crystal_orientation_model.reciprocal_matrix())
partiality_model_flex = ph.calc_partiality_anisotropy_set(a_star_model, observations_original.indices(), ry, rz, re, two_theta_flex, alpha_angle_set)
partiality_model = partiality_model_flex.as_numpy_array()
I_o_model = G * np.exp(-2*B*sin_theta_over_lambda_sq) * partiality_model * I_r_true
SE_of_the_estimate = standard_error_of_the_estimate(I_o_true/sigI_o_true, I_o_model/sigI_o_true, len(x0))
R_sq = coefficient_of_determination(I_o_true/sigI_o_true, I_o_model/sigI_o_true)*100
CC = np.corrcoef(I_o_true/sigI_o_true, I_o_model/sigI_o_true)[0,1]
var_I_p = ((observations_original.sigmas()/observations_original.data())**2).as_numpy_array()
#calculate standard error for the parameters
if cov_x is None:
se_xopt = np.array([0,0,0,0,0,0,0,0,0,0,0,0,0])
var_k = flex.double([0]*len(I_o_model))
var_p = flex.double([0]*len(I_o_model))
SE_I = np.sqrt(var_I_p)*(I_o_model)
else:
pcov = cov_x * (SE_of_the_estimate**2)
var_xopt_limit = np.array([pcov[i,i] for i in range(len(x0))])
var_xopt = prep_variance(var_xopt_limit, cs)
se_xopt = np.sqrt(var_xopt)
#calculate error propagation in I_full
se_G, se_B, se_rotx, se_roty, se_ry, se_rz, se_re, se_a, se_b, se_c, se_alpha, se_beta, se_gamma = se_xopt
###k = G_0 * exp(-2*B*sin_theta_over_lambda_sq)
se_k_sq = (np.exp(-2*B*sin_theta_over_lambda_sq)*se_G)**2 + (-2 * G * sin_theta_over_lambda_sq * np.exp(-2*B*sin_theta_over_lambda_sq) * se_B)**2
var_k = se_k_sq/((G*np.exp(-2*B*sin_theta_over_lambda_sq))**2)
###use finite differences to propagate errors in partility function p.
var_p_sq = flex.double()
fmode_arr = ('G','B','rotx','roty','ry','rz','re','a','b','c','alpha','beta','gamma')
for miller_index, bragg_angle, alpha_angle in zip(observations_original.indices(), two_theta_flex, alpha_angle_set):
Dp = flex.double()
cn_fmode = 0
for fmode in fmode_arr:
dp = misc.derivative(func_partiality, xopt[cn_fmode], args=(miller_index,
crystal_init_orientation.crystal_rotation_matrix(), wavelength, (bragg_angle, alpha_angle), xopt, fmode))
cn_fmode += 1
Dp.append(dp)
se_p_sq = 0
for dp, se in zip(Dp, se_xopt):
se_p_sq += (dp*se)**2
var_p_sq.append(se_p_sq)
var_p = (var_p_sq/(partiality_model_flex**2)).as_numpy_array()
SE_I = np.sqrt(var_I_p + var_k + var_p)*(I_o_model)
"""
print 'Predictor'
print 'G %.4g'%(G)
print 'B-factor %.4g'%(B)
print 'rotx %.4g'%(rotx*180/math.pi)
print 'roty %.4g'%(roty*180/math.pi)
print 'ry %.4g'%(ry)
print 'rz %.4g'%(rz)
print 're %.4g'%(re)
print 'uc', uc_opt
print 'S = %.4g'%SE_of_the_estimate
print 'R-Sq = %.4g%%'%(R_sq)
print 'CC = %.4g'%(CC)
plt.scatter(I_r_true, I_o_true,s=10, marker='x', c='r')
plt.scatter(I_r_true, I_o_model,s=10, marker='o', c='b')
plt.title('G=%.4g B=%.4g rotx=%.4g roty=%.4g R-sq=%.4g%%'%(xopt[0], xopt[1], xopt[2]*180/math.pi, xopt[3]*180/math.pi, R_sq))
plt.xlabel('I_ref')
plt.ylabel('I_obs')
plt.show()
"""
return xopt, se_xopt, (SE_of_the_estimate, R_sq, CC), partiality_model, SE_I, var_I_p, var_k, var_p
def postrefine_by_frame(self, frame_no, pickle_filename, iparams,
miller_array_ref, pres_in, avg_mode):
#1. Prepare data
observations_pickle = read_frame(pickle_filename)
pickle_filepaths = pickle_filename.split('/')
img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1]
txt_exception = ' {0:40} ==> '.format(img_filename_only)
inputs, txt_organize_input = self.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pickle_filename)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs
else:
txt_exception += txt_organize_input + '\n'
return None, txt_exception
#2. Select data for post-refinement (only select indices that are common with the reference set
observations_non_polar, index_basis_name = self.get_observations_non_polar(
observations_original, pickle_filename, iparams)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_ref.indices(),
miller_indices=observations_non_polar.indices())
pair_0 = flex.size_t([pair[0] for pair in matches.pairs()])
pair_1 = flex.size_t([pair[1] for pair in matches.pairs()])
references_sel = miller_array_ref.select(pair_0)
observations_original_sel = observations_original.select(pair_1)
observations_non_polar_sel = observations_non_polar.select(pair_1)
alpha_angle_set = alpha_angle.select(pair_1)
spot_pred_x_mm_set = spot_pred_x_mm.select(pair_1)
spot_pred_y_mm_set = spot_pred_y_mm.select(pair_1)
#4. Do least-squares refinement
lsqrh = leastsqr_handler()
try:
refined_params, stats, n_refl_postrefined = lsqrh.optimize(
references_sel.data(), observations_original_sel, wavelength,
crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set,
spot_pred_y_mm_set, iparams, pres_in,
observations_non_polar_sel, detector_distance_mm)
except Exception:
txt_exception += 'optimization failed.\n'
return None, txt_exception
#caculate partiality for output (with target_anomalous check)
G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \
a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params
inputs, txt_organize_input = self.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pickle_filename)
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs
observations_non_polar, index_basis_name = self.get_observations_non_polar(
observations_original, pickle_filename, iparams)
from cctbx.uctbx import unit_cell
uc_fin = unit_cell(
(a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin))
if pres_in is not None:
crystal_init_orientation = pres_in.crystal_orientation
two_theta = observations_original.two_theta(
wavelength=wavelength).data()
ph = partiality_handler()
partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(
uc_fin, rotx_fin, roty_fin, observations_original.indices(),
ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, two_theta,
alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm,
spot_pred_y_mm, detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
#calculate the new crystal orientation
O = sqr(uc_fin.orthogonalization_matrix()).transpose()
R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose()
from cctbx.crystal_orientation import crystal_orientation, basis_type
CO = crystal_orientation(O * R, basis_type.direct)
crystal_fin_orientation = CO.rotate_thru(
(1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin)
#remove reflections with partiality below threshold
i_sel = partiality_fin > iparams.merge.partiality_min
partiality_fin_sel = partiality_fin.select(i_sel)
rs_fin_sel = rs_fin.select(i_sel)
rh_fin_sel = rh_fin.select(i_sel)
observations_non_polar_sel = observations_non_polar.customized_copy(\
indices=observations_non_polar.indices().select(i_sel),
data=observations_non_polar.data().select(i_sel),
sigmas=observations_non_polar.sigmas().select(i_sel))
observations_original_sel = observations_original.customized_copy(\
indices=observations_original.indices().select(i_sel),
data=observations_original.data().select(i_sel),
sigmas=observations_original.sigmas().select(i_sel))
pres = postref_results()
pres.set_params(observations=observations_non_polar_sel,
observations_original=observations_original_sel,
refined_params=refined_params,
stats=stats,
partiality=partiality_fin_sel,
rs_set=rs_fin_sel,
rh_set=rh_fin_sel,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_fin_orientation,
detector_distance_mm=detector_distance_mm)
r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100
r_xy_change = (
(pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100
cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100
txt_postref = '{0:40} => RES:{1:5.2f} NREFL:{2:5d} R:{3:6.1f}% RXY:{4:5.1f}% CC:{5:5.1f}% G:{6:6.4f} B:{7:5.1f} CELL:{8:6.1f}{9:6.1f} {10:6.1f} {11:5.1f} {12:5.1f} {13:5.1f}'.format(
img_filename_only + ' (' + index_basis_name + ')',
observations_original_sel.d_min(),
len(observations_original_sel.data()), r_change, r_xy_change,
cc_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin,
beta_fin, gamma_fin)
print txt_postref
txt_postref += '\n'
return pres, txt_postref
def scale_frame_by_mean_I(self, frame_no, pickle_filename, iparams, mean_of_mean_I, avg_mode='average'):
observations_pickle = pickle.load(open(pickle_filename,"rb"))
pickle_filepaths = pickle_filename.split('/')
img_filename_only = pickle_filepaths[len(pickle_filepaths)-1]
inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename)
txt_exception = ' {0:40} ==> '.format(img_filename_only)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
else:
txt_exception += txt_organize_input + '\n'
return None, txt_exception
wavelength = observations_pickle["wavelength"]
crystal_init_orientation = observations_pickle["current_orientation"][0]
#select only reflections matched with scale input params.
#filter by resolution
i_sel_res = observations_original.resolution_filter_selection(d_min=iparams.scale.d_min,
d_max=iparams.scale.d_max)
observations_original_sel = observations_original.select(i_sel_res)
alpha_angle_sel = alpha_angle.select(i_sel_res)
spot_pred_x_mm_sel = spot_pred_x_mm.select(i_sel_res)
spot_pred_y_mm_sel = spot_pred_y_mm.select(i_sel_res)
#filter by sigma
i_sel_sigmas = (observations_original_sel.data()/observations_original_sel.sigmas()) > iparams.scale.sigma_min
observations_original_sel = observations_original_sel.select(i_sel_sigmas)
alpha_angle_sel = alpha_angle_sel.select(i_sel_sigmas)
spot_pred_x_mm_sel = spot_pred_x_mm_sel.select(i_sel_sigmas)
spot_pred_y_mm_sel = spot_pred_y_mm_sel.select(i_sel_sigmas)
observations_non_polar_sel, index_basis_name = self.get_observations_non_polar(observations_original_sel, pickle_filename, iparams)
observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams)
uc_params = observations_original.unit_cell().parameters()
ph = partiality_handler()
r0 = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()),
observations_original_sel.indices(), wavelength)
#calculate first G
(G, B) = (1,0)
stats = (0,0,0,0,0,0,0,0,0,0)
if mean_of_mean_I > 0:
G = flex.median(observations_original_sel.data())/mean_of_mean_I
if iparams.flag_apply_b_by_frame:
try:
from mod_util import mx_handler
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(iparams.n_residues)
observations_as_f = observations_non_polar_sel.as_amplitude_array()
binner_template_asu = observations_as_f.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True)
G = wp.wilson_intensity_scale_factor*1e3
B = wp.wilson_b
except Exception:
txt_exception += 'warning B-factor calculation failed.\n'
return None, txt_exception
two_theta = observations_original.two_theta(wavelength=wavelength).data()
sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data()
ry, rz, re, voigt_nu, rotx, roty = (0, 0, iparams.gamma_e, iparams.voigt_nu, 0, 0)
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
crystal_init_orientation.unit_cell(),
rotx, roty, observations_original.indices(),
ry, rz, r0, re, voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm,
detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
if iparams.flag_plot_expert:
n_bins = 20
binner = observations_original.setup_binner(n_bins=n_bins)
binner_indices = binner.bin_indices()
avg_partiality_init = flex.double()
avg_rs_init = flex.double()
avg_rh_init = flex.double()
one_dsqr_bin = flex.double()
for i in range(1,n_bins+1):
i_binner = (binner_indices == i)
if len(observations_original.data().select(i_binner)) > 0:
print binner.bin_d_range(i)[1], flex.mean(partiality_init.select(i_binner)), flex.mean(rs_init.select(i_binner)), flex.mean(rh_init.select(i_binner)), len(partiality_init.select(i_binner))
#save results
refined_params = flex.double([G,B,rotx,roty,ry,rz,r0,re,voigt_nu,uc_params[0],uc_params[1],uc_params[2],uc_params[3],uc_params[4],uc_params[5]])
pres = postref_results()
pres.set_params(observations = observations_non_polar,
observations_original = observations_original,
refined_params=refined_params,
stats=stats,
partiality=partiality_init,
rs_set=rs_init,
rh_set=rh_init,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_init_orientation,
detector_distance_mm=detector_distance_mm,
identified_isoform=identified_isoform,
mapped_predictions=mapped_predictions,
xbeam=xbeam,
ybeam=ybeam)
txt_scale_frame_by_mean_I = ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} G:{3:10.3e} B:{4:7.1f} CELL:{5:6.2f} {6:6.2f} {7:6.2f} {8:6.2f} {9:6.2f} {10:6.2f}'.format(img_filename_only+' ('+index_basis_name+')', observations_original.d_min(), len(observations_original_sel.data()), G, B, uc_params[0],uc_params[1],uc_params[2],uc_params[3],uc_params[4],uc_params[5])
print txt_scale_frame_by_mean_I
txt_scale_frame_by_mean_I += '\n'
return pres, txt_scale_frame_by_mean_I
def func(self, params, args):
I_r = args[0]
miller_array_o = args[1]
wavelength = args[2]
alpha_angle_set = args[3]
crystal_init_orientation = args[4]
spot_pred_x_mm_set = args[5]
spot_pred_y_mm_set = args[6]
detector_distance_mm = args[7]
refine_mode = args[8]
const_params = args[9]
b0 = args[10]
miller_array_iso = args[11]
iparams = args[12]
partiality_model = iparams.partiality_model
flag_volume_correction = iparams.flag_volume_correction
flag_beam_divergence = iparams.flag_beam_divergence
b_refine_d_min = iparams.b_refine_d_min
I_o = miller_array_o.data()
sigI_o = miller_array_o.sigmas()
miller_indices_original = miller_array_o.indices()
sin_theta_over_lambda_sq = miller_array_o.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data()
two_theta_flex = miller_array_o.two_theta(wavelength=wavelength).data()
cs = miller_array_o.crystal_symmetry().space_group().crystal_system()
if refine_mode == 'scale_factor':
G, B = params
rotx, roty, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = const_params
elif refine_mode == 'crystal_orientation':
rotx, roty = params
G, B, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = const_params
elif refine_mode == 'reflecting_range':
ry, rz, r0, re, nu = params
G, B, rotx, roty, a, b, c, alpha, beta, gamma = const_params
elif refine_mode == 'unit_cell':
a, b, c, alpha, beta, gamma = self.prep_output(params, cs)
G, B, rotx, roty, ry, rz, r0, re, nu = const_params
elif refine_mode == 'allparams':
a, b, c, alpha, beta, gamma = self.prep_output(params[7:], cs)
rotx, roty, ry, rz, r0, re, nu = params[0:7]
G,B = const_params
try:
uc = unit_cell((a,b,c,alpha,beta,gamma))
except Exception:
return None
G,B,rotx,roty,ry,rz,r0,re,nu,a,b,c,alpha,beta,gamma = flex.double([G,B,rotx,roty,ry,rz,r0,re,nu,a,b,c,alpha,beta,gamma])
ph = partiality_handler()
p_calc_flex, delta_xy_flex, rs_set, dummy = ph.calc_partiality_anisotropy_set(\
uc, rotx, roty, miller_indices_original, ry, rz, r0, re, nu, two_theta_flex,
alpha_angle_set, wavelength, crystal_init_orientation, spot_pred_x_mm_set,
spot_pred_y_mm_set, detector_distance_mm, partiality_model, flag_beam_divergence)
if miller_array_o.d_min() < b_refine_d_min:
I_o_full = ph.calc_full_refl(I_o, sin_theta_over_lambda_sq, G, B, p_calc_flex, rs_set, flag_volume_correction)
else:
I_o_full = ph.calc_full_refl(I_o, sin_theta_over_lambda_sq, G, b0, p_calc_flex, rs_set, flag_volume_correction)
if refine_mode == 'unit_cell':
error = delta_xy_flex
else:
error = ((I_r - I_o_full)/sigI_o)
"""
import math
print refine_mode, 'G:%.4g B:%.4g rotx:%.4g roty:%.4g r0:%.4g re:%.4g nu:%6.4f a:%.4g b:%.4g c:%.4g fpr:%.4g fxy:%.4g n_refl=%5.0f'%(G, B, rotx*180/math.pi, roty*180/math.pi, r0, re, nu, a, b, c, flex.sum(((I_r - I_o_full)/sigI_o)**2), flex.sum(delta_xy_flex**2), len(I_o_full))
"""
return error
def postrefine_by_frame(self, frame_no, pres_in, iparams, miller_array_ref,
avg_mode):
#Prepare data
if pres_in is None:
return None, 'Found empty pickle file'
observations_pickle = pickle.load(open(pres_in.pickle_filename, "rb"))
wavelength = observations_pickle["wavelength"]
crystal_init_orientation = observations_pickle["current_orientation"][
0]
pickle_filename = pres_in.pickle_filename
pickle_filepaths = pickle_filename.split('/')
img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1]
txt_exception = ' {0:40} ==> '.format(img_filename_only)
inputs, txt_organize_input = self.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pickle_filename)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
else:
txt_exception += txt_organize_input + '\n'
return None, txt_exception
#Select data for post-refinement (only select indices that are common with the reference set
observations_non_polar, index_basis_name = self.get_observations_non_polar(
observations_original, pickle_filename, iparams)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_ref.indices(),
miller_indices=observations_non_polar.indices())
I_ref_match = flex.double(
[miller_array_ref.data()[pair[0]] for pair in matches.pairs()])
miller_indices_ref_match = flex.miller_index(
(miller_array_ref.indices()[pair[0]] for pair in matches.pairs()))
I_obs_match = flex.double([
observations_non_polar.data()[pair[1]] for pair in matches.pairs()
])
sigI_obs_match = flex.double([
observations_non_polar.sigmas()[pair[1]]
for pair in matches.pairs()
])
miller_indices_original_obs_match = flex.miller_index((observations_original.indices()[pair[1]] \
for pair in matches.pairs()))
miller_indices_non_polar_obs_match = flex.miller_index((observations_non_polar.indices()[pair[1]] \
for pair in matches.pairs()))
alpha_angle_set = flex.double(
[alpha_angle[pair[1]] for pair in matches.pairs()])
spot_pred_x_mm_set = flex.double(
[spot_pred_x_mm[pair[1]] for pair in matches.pairs()])
spot_pred_y_mm_set = flex.double(
[spot_pred_y_mm[pair[1]] for pair in matches.pairs()])
references_sel = miller_array_ref.customized_copy(
data=I_ref_match, indices=miller_indices_ref_match)
observations_original_sel = observations_original.customized_copy(
data=I_obs_match,
sigmas=sigI_obs_match,
indices=miller_indices_original_obs_match)
observations_non_polar_sel = observations_non_polar.customized_copy(
data=I_obs_match,
sigmas=sigI_obs_match,
indices=miller_indices_non_polar_obs_match)
#Do least-squares refinement
lsqrh = leastsqr_handler()
try:
refined_params, stats, n_refl_postrefined = lsqrh.optimize(
I_ref_match, observations_original_sel, wavelength,
crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set,
spot_pred_y_mm_set, iparams, pres_in,
observations_non_polar_sel, detector_distance_mm)
except Exception:
txt_exception += 'optimization failed.\n'
return None, txt_exception
#caculate partiality for output (with target_anomalous check)
G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \
a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params
inputs, txt_organize_input = self.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pickle_filename)
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
observations_non_polar, index_basis_name = self.get_observations_non_polar(
observations_original, pickle_filename, iparams)
from cctbx.uctbx import unit_cell
uc_fin = unit_cell(
(a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin))
crystal_init_orientation = pres_in.crystal_orientation
two_theta = observations_original.two_theta(
wavelength=wavelength).data()
ph = partiality_handler()
partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(
uc_fin, rotx_fin, roty_fin, observations_original.indices(),
ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, two_theta,
alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm,
spot_pred_y_mm, detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
#calculate the new crystal orientation
O = sqr(uc_fin.orthogonalization_matrix()).transpose()
R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose()
from cctbx.crystal_orientation import crystal_orientation, basis_type
CO = crystal_orientation(O * R, basis_type.direct)
crystal_fin_orientation = CO.rotate_thru(
(1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin)
#remove reflections with partiality below threshold
i_sel = partiality_fin > iparams.merge.partiality_min
partiality_fin_sel = partiality_fin.select(i_sel)
rs_fin_sel = rs_fin.select(i_sel)
rh_fin_sel = rh_fin.select(i_sel)
observations_non_polar_sel = observations_non_polar.select(i_sel)
observations_original_sel = observations_original.select(i_sel)
mapped_predictions = mapped_predictions.select(i_sel)
pres = postref_results()
pres.set_params(observations=observations_non_polar_sel,
observations_original=observations_original_sel,
refined_params=refined_params,
stats=stats,
partiality=partiality_fin_sel,
rs_set=rs_fin_sel,
rh_set=rh_fin_sel,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_init_orientation,
detector_distance_mm=detector_distance_mm,
identified_isoform=identified_isoform,
mapped_predictions=mapped_predictions,
xbeam=xbeam,
ybeam=ybeam)
r_change, r_xy_change, cc_change, cc_iso_change = (0, 0, 0, 0)
try:
r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100
r_xy_change = (
(pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100
cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100
cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init) /
pres.CC_iso_init) * 100
except Exception:
pass
txt_postref = ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}'.format(
img_filename_only + ' (' + index_basis_name + ')',
observations_original_sel.d_min(),
len(observations_original_sel.data()), r_change, r_xy_change,
cc_change, cc_iso_change, pres.G, pres.B, a_fin, b_fin, c_fin,
alpha_fin, beta_fin, gamma_fin)
print txt_postref
txt_postref += '\n'
return pres, txt_postref
