def calc_partiality_mproc(i_sel, observations, crystal_init_orientation): h = observations.indices()[i_sel] partiality_c_set = flex.double() a_star_init = sqr(crystal_init_orientation.reciprocal_matrix()) eh = energy_handler() eh.get_energy_info(file_name_in_img, file_name_in_energy, pickle_filename) #calculate spot_radisu (rs) from mean_wavelength spot_radius = calc_spot_radius(a_star_init, observations.indices(), eh.mean_wavelength) for wavelength, c_weight in zip(eh.wavelength_at_counts, eh.photon_counts_normalized): ph = partiality_handler(wavelength, spot_radius) p, dummy = ph.calc_partiality(a_star_init, h) partiality_c_set.append(p*c_weight) #correct intensity usinging single color ph = partiality_handler(eh.mean_wavelength, spot_radius) p_single, dummy = ph.calc_partiality(a_star_init, h) I_p_single = observations.data()[i_sel]/p_single sigI_p_single = observations.sigmas()[i_sel]/p_single p_multi = sum(partiality_c_set)/sum(eh.photon_counts_normalized) I_p_multi = observations.data()[i_sel]/p_multi sigI_p_multi = observations.sigmas()[i_sel]/p_multi return i_sel, h, p_single, I_p_single, sigI_p_single, p_multi, I_p_multi, sigI_p_multi
def prepare_data_microcycle(self, refine_mode, iparams, observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, I_r_flex, init_params, crystal_init_orientation, wavelength, detector_distance_mm): #prepare data if refine_mode == 'crystal_orientation': pr_d_min = iparams.postref.crystal_orientation.d_min pr_d_max = iparams.postref.crystal_orientation.d_max pr_sigma_min = iparams.postref.crystal_orientation.sigma_min pr_partiality_min = iparams.postref.crystal_orientation.partiality_min pr_uc_tol = iparams.postref.unit_cell.uc_tolerance elif refine_mode == 'reflecting_range': pr_d_min = iparams.postref.reflecting_range.d_min pr_d_max = iparams.postref.reflecting_range.d_max pr_sigma_min = iparams.postref.reflecting_range.sigma_min pr_partiality_min = iparams.postref.reflecting_range.partiality_min pr_uc_tol = iparams.postref.unit_cell.uc_tolerance elif refine_mode == 'unit_cell': pr_d_min = iparams.postref.unit_cell.d_min pr_d_max = iparams.postref.unit_cell.d_max pr_sigma_min = iparams.postref.unit_cell.sigma_min pr_partiality_min = iparams.postref.unit_cell.partiality_min pr_uc_tol = iparams.postref.unit_cell.uc_tolerance elif refine_mode == 'allparams': pr_d_min = iparams.postref.allparams.d_min pr_d_max = iparams.postref.allparams.d_max pr_sigma_min = iparams.postref.allparams.sigma_min pr_partiality_min = iparams.postref.allparams.partiality_min pr_uc_tol = iparams.postref.unit_cell.uc_tolerance #filter by resolution observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, I_r_flex) #filter by sigma observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel) #extract refined parameters G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = init_params #filter by partiality two_theta = observations_original_sel.two_theta( wavelength=wavelength).data() uc = unit_cell((a, b, c, alpha, beta, gamma)) ph = partiality_handler() partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set( uc, rotx, roty, observations_original_sel.indices(), ry, rz, r0, re, voigt_nu, two_theta, alpha_angle_sel, wavelength, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'partiality', [pr_partiality_min], observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel, partiality_in=partiality_init) return observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel
def compute_cost_refine_crystal(self, I_ref, observations_original_sel, crystal_init_orientation, wavelength, parameters): G = parameters[0] B_factor = parameters[1] rotx = parameters[2] roty = parameters[3] ry = parameters[4] rz = parameters[5] I_obs = observations_original_sel.data() sigI_obs = observations_original_sel.sigmas() observations_original_sel_two_theta = observations_original_sel.two_theta(wavelength=wavelength) two_theta = observations_original_sel_two_theta.data() sin_theta_over_lambda_sq = observations_original_sel_two_theta.sin_theta_over_lambda_sq().data() effective_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx ).rotate_thru((0,1,0),roty) effective_a_star = sqr(effective_orientation.reciprocal_matrix()) ph = partiality_handler(wavelength, 0) partiality = ph.calc_partiality_anisotropy_set(effective_a_star, observations_original_sel.indices(), ry, rz, two_theta) excursions = (((G * (flex.exp(-2*B_factor*sin_theta_over_lambda_sq)) * I_obs)/partiality) - I_ref) / sigI_obs return excursions
def jacobian_callable(pfh,current_values): ry = current_values[0] rz = current_values[1] G = scale_factors[0] B_factor = scale_factors[1] rotx = rotations[0] roty = rotations[1] I_obs = observations_original_sel.data() sigI_obs = observations_original_sel.sigmas() observations_original_sel_two_theta = observations_original_sel.two_theta(wavelength=wavelength) two_theta = observations_original_sel_two_theta.data() sin_theta_over_lambda_sq = observations_original_sel_two_theta.sin_theta_over_lambda_sq().data() delta_r = 0.00000001 #0. Calculate current partiality based on current rotx crystal_current_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx ).rotate_thru((0,1,0),roty) a_star = sqr(crystal_current_orientation.reciprocal_matrix()) ph = partiality_handler(wavelength, 0) pDg_pDry = flex.double() pDg_pDrz = flex.double() for j in range(len(observations_original_sel.indices())): miller_index = observations_original_sel.indices()[j] p, rh, rs = ph.calc_partiality_anisotropy(a_star, miller_index, ry, rz, two_theta[j]) pDg_pDP = (-1 * G * (math.exp(-2*B_factor*sin_theta_over_lambda_sq[j])) * I_obs[j])/(sigI_obs[j]*(p**2)) pDP_pDrs = (4*rs*(rh**2))/(((2*(rh**2))+(rs**2))**2) rs_as_k = math.sqrt(((ry * math.cos(two_theta[j]))**2)+((rz * math.sin(two_theta[j]))**2)) pDrs_pDry = ry * (math.cos(two_theta[j])**2)/ rs_as_k pDrs_pDrz = rz * (math.sin(two_theta[j])**2)/ rs_as_k pDg_pDry.append(pDg_pDP * pDP_pDrs * pDrs_pDry) pDg_pDrz.append(pDg_pDP * pDP_pDrs * pDrs_pDrz) """ #checking partial derivatives g_now = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx,roty,ry,rz)) g_delta_ry = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx,roty,ry+delta_r,rz)) g_delta_rz = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx,roty,ry,rz+delta_r)) pDg_pDry_fd = (g_delta_ry - g_now)/ delta_r pDg_pDrz_fd = (g_delta_rz - g_now)/ delta_r print sum(flex.abs(pDg_pDry_fd)), sum(flex.abs(pDg_pDry_fd-pDg_pDry)) print sum(flex.abs(pDg_pDrz_fd)), sum(flex.abs(pDg_pDrz_fd-pDg_pDrz)) print """ return pDg_pDry, pDg_pDrz
def calc_partiality_mproc(calc_partiality_arg, frames): calc_partiality_arg_tag = calc_partiality_arg.split(':') frame_no = int(calc_partiality_arg_tag[0]) ry_shift = float(calc_partiality_arg_tag[1]) pickle_filename = frames[frame_no] trial_results = pickle.load(open(pickle_filename,"rb")) crystal_init_orientation = trial_results["current_orientation"][0] wavelength = trial_results["wavelength"] observations = trial_results["observations"][0] crystal_pointgroup = trial_results["pointgroup"] unit_cell = trial_results["current_orientation"][0].unit_cell() target_unit_cell = unit_cell.parameters() a_star_init = sqr(crystal_init_orientation.reciprocal_matrix()) eh = energy_handler() eh.get_energy_info(file_name_in_img, file_name_in_energy, pickle_filename) #calc spot_radius y and z component spot_radius = calc_spot_radius(a_star_init, observations.indices(), wavelength) ry = spot_radius + ry_shift rz = ry * 0.5 #calc alpha_angle two_theta = observations.two_theta(wavelength=wavelength).data() ph = partiality_handler(wavelength, spot_radius) I_p_aniso = flex.double() sigI_p_aniso = flex.double() I_p = flex.double() sigI_p = flex.double() for i_sel in range(len(observations.indices())): h = observations.indices()[i_sel] p, dummy = ph.calc_partiality(a_star_init, h) I_p.append(observations.data()[i_sel]/ p) sigI_p.append(observations.sigmas()[i_sel]/ p) alpha_angle = two_theta[i_sel] p_aniso, rs_aniso = ph.calc_partiality_anisotropy(a_star_init, h, ry, rz, alpha_angle) I_p_aniso.append(observations.data()[i_sel]/ p_aniso) sigI_p_aniso.append(observations.sigmas()[i_sel]/ p_aniso) observations_p = observations.customized_copy(data=I_p, sigmas=sigI_p) observations_p_aniso = observations.customized_copy(data=I_p_aniso, sigmas=sigI_p_aniso) return frame_no, observations_p, observations_p_aniso, (ry, rz, spot_radius)
def func_partiality(x, *args): miller_index = args[0] crystal_rotation_matrix = args[1] wavelength = args[2] bragg_angle, alpha_angle = args[3] const_params = args[4] fmode = args[5] G, B, rotx, roty, ry, rz, re, a, b, c, alpha, beta, gamma = const_params if fmode == 'G': G = x elif fmode== 'B': B = x elif fmode== 'rotx': rotx = x elif fmode== 'roty': roty = x elif fmode== 'ry': ry = x elif fmode== 'rz': rz = x elif fmode== 're': re = x elif fmode== 'a': a = x elif fmode== 'b': b = x elif fmode== 'c': c = x elif fmode== 'alpha': alpha = x elif fmode== 'beta': beta = x elif fmode== 'gamma': gamma = x uc = unit_cell((a,b,c,alpha,beta,gamma)) crystal_init_orientation = get_crystal_orientation(uc.orthogonalization_matrix(), crystal_rotation_matrix) crystal_orientation_model = crystal_init_orientation.rotate_thru((1,0,0), rotx ).rotate_thru((0,1,0), roty) a_star_model = sqr(crystal_orientation_model.reciprocal_matrix()) ph = partiality_handler(wavelength, 0) partiality, dummy, dummy = ph.calc_partiality_anisotropy(a_star_model, miller_index, ry, rz, re, bragg_angle, alpha_angle) return partiality
def calc_full_observations(self, observations_type='asymmetric_unit', flag_apply_partiality=False): """ calculate partiality and full observations (miller array) """ if True: if self.init_type == 'from_file': partiality_hdlr = partiality_handler(self.iparams) self.partiality, dum0, self.rh_set, self.rs_set = partiality_hdlr.calc_partiality_anisotropy_set( self) b_cart = sqr((self.b11, self.b12, self.b13, self.b12, self.b22, self.b23, self.b13, self.b23, self.b33)) dw = self.calc_debye_waller_factor(self.observations, b_cart) if flag_apply_partiality: partiality = self.partiality[:] else: partiality = flex.double([1] * len(self.observations.data())) G_set = flex.double([self.G] * len(self.observations.data())) i_lowres = self.observations.resolution_filter_selection( d_min=self.scale_0_d_max, d_max=self.iparams.d_max) G_set.set_selected(i_lowres, self.G_lowres) dw.set_selected(i_lowres, 1.0) I_full = flex.double(self.observations.data() / (G_set * dw * partiality)) sigI_full = flex.double(self.observations.sigmas() / (G_set * dw * partiality)) elif self.init_type == 'from_merge': I_full = self.observations.data()[:] sigI_full = self.observations.sigmas()[:] if observations_type == 'asymmetric_unit': observations_full = self.observations.customized_copy( data=I_full, sigmas=sigI_full) elif observations_type == 'original': observations_full = self.observations_original.customized_copy( data=I_full, sigmas=sigI_full) return observations_full
def compute_cost(self, I_ref, I_obs, sigI_obs, crystal_init_orientation, miller_indices_ori, wavelength, current_values): G = current_values[0] rotx = current_values[1] roty = current_values[2] rotz = current_values[3] rs = current_values[4] m = len(I_ref) effective_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx ).rotate_thru((0,1,0),roty ).rotate_thru((0,0,1),rotz) effective_a_star = sqr(effective_orientation.reciprocal_matrix()) ph = partiality_handler(wavelength, rs) partiality = flex.double([ph.calc_partiality(effective_a_star, miller_index)[0] for miller_index in miller_indices_ori]) J = flex.sum(((((G * I_obs)/partiality)- I_ref)/ sigI_obs)**2)/(2*m) return J
def fvec_callable(pfh,current_values): rotx = current_values[0] roty = current_values[1] ry = spot_radius rz = spot_radius G = scale_factors[0] B_factor = scale_factors[1] I_obs = observations_original_sel.data() sigI_obs = observations_original_sel.sigmas() observations_original_sel_two_theta = observations_original_sel.two_theta(wavelength=wavelength) two_theta = observations_original_sel_two_theta.data() sin_theta_over_lambda_sq = observations_original_sel_two_theta.sin_theta_over_lambda_sq().data() effective_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx ).rotate_thru((0,1,0),roty) effective_a_star = sqr(effective_orientation.reciprocal_matrix()) ph = partiality_handler(wavelength, 0) partiality = ph.calc_partiality_anisotropy_set(effective_a_star, observations_original_sel.indices(), ry, rz, two_theta) excursions = (((G * (flex.exp(-2*B_factor*sin_theta_over_lambda_sq)) * I_obs)/partiality) - I_ref) / sigI_obs corr_now, slope_now = get_overall_correlation((G * (flex.exp(-2*B_factor*sin_theta_over_lambda_sq)) * I_obs)/partiality, I_ref) """ print "ROTATION G=%5.3f B_factor=%5.3f rotx=%6.5f roty=%6.5f ry=%6.5f rz=%6.5f J=%6.3f cc=%6.3f slope=%6.3f p_mean=%6.3f"% \ (G, B_factor, rotx*180/math.pi, roty*180/math.pi, ry, rz, sum(excursions**2), corr_now, slope_now, flex.mean(partiality)) plt.scatter(I_ref,(G * (flex.exp(-2*B_factor*sin_theta_over_lambda_sq)) * I_obs)/partiality,s=10, marker='x', c='r') plt.title('J=%6.5f CC=%6.5f Slope=%6.5f'%(sum(excursions**2), corr_now, slope_now)) plt.xlabel('Reference intensity') plt.ylabel('Observed intensity (scaled)') plt.show() """ return excursions
assume_index_matching=True) print 'Overall R-factor:', r1_factor #calculate r_factor for partiality-corrected scaled data pixel_size_mm = 0.079346 crystal_init_orientation = observations_pickle["current_orientation"][0] wavelength = observations_pickle["wavelength"] detector_distance_mm = observations_pickle['distance'] mm_predictions = pixel_size_mm * ( observations_pickle['mapped_predictions'][0]) xbeam = observations_pickle["xbeam"] ybeam = observations_pickle["ybeam"] alpha_angle_obs = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \ for pred in mm_predictions]) spot_pred_x_mm = flex.double([pred[0] - xbeam for pred in mm_predictions]) spot_pred_y_mm = flex.double([pred[1] - ybeam for pred in mm_predictions]) ph = partiality_handler() r0 = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()), observations.indices(), wavelength) ry, rz, re, voigt_nu, rotx, roty = (0, 0, 0.003, 0.5, 0, 0) partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\ crystal_init_orientation.unit_cell(), rotx, roty, observations.indices(), ry, rz, r0, re, voigt_nu, two_theta, alpha_angle_obs, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, "Voigt", False) I_full = observations.data() / partiality_init sigI_full = observations.sigmas() / partiality_init obs_full = observations.customized_copy(data=I_full, sigmas=sigI_full)
def optimize_scalefactors(self, I_r_flex, observations_original, wavelength, crystal_init_orientation, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams, pres_in, observations_non_polar, detector_distance_mm, const_params): ph = partiality_handler() pr_d_min = iparams.postref.scale.d_min pr_d_max = iparams.postref.scale.d_max pr_sigma_min = iparams.postref.scale.sigma_min pr_partiality_min = iparams.postref.scale.partiality_min #filter by resolution observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,\ spot_pred_x_mm, spot_pred_y_mm, I_r_flex) #filter by sigma observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,\ spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel) I_r_true = I_ref_sel[:] I_o_true = observations_original_sel.data()[:] if pres_in is not None: G, B, b0 = pres_in.G, pres_in.B, pres_in.B else: G = flex.median(I_o_true)/flex.median(I_r_true) B,b0 = (0,0) if iparams.flag_apply_b_by_frame: try: from mod_util import mx_handler mxh = mx_handler() asu_contents = mxh.get_asu_contents(iparams.n_residues) observations_as_f = observations_original_sel.as_amplitude_array() binner_template_asu = observations_as_f.setup_binner(auto_binning=True) wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True) G = wp.wilson_intensity_scale_factor*1e3 B = wp.wilson_b except Exception: pass refine_mode = 'scale_factor' xinp = flex.double([G,B]) args = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, refine_mode, const_params, b0, None, iparams) lh = lbfgs_handler(current_x=xinp, args=args) G_fin, B_fin = (lh.x[0], lh.x[1]) rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = const_params two_theta = observations_original.two_theta(wavelength=wavelength) sin_theta_over_lambda_sq = two_theta.sin_theta_over_lambda_sq().data() uc = unit_cell((a,b,c,alpha,beta,gamma)) ph = partiality_handler() partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty, observations_original.indices(), ry, rz, r0, re, voigt_nu, two_theta.data(), alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, G, B, partiality_init, rs_init) I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, G_fin, B_fin, partiality_init, rs_init) SE_of_the_estimate = standard_error_of_the_estimate(I_r_flex, I_o_fin, 2) R_sq = coefficient_of_determination(I_r_flex,I_o_fin)*100 CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient() CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient() err_init = (I_r_flex - I_o_init)/observations_original.sigmas() R_init = math.sqrt(flex.sum(err_init**2)) err_final = (I_r_flex - I_o_fin)/observations_original.sigmas() R_final = math.sqrt(flex.sum(err_final**2)) R_xy_init = 0 R_xy_final = 0 CC_iso_init = 0 CC_iso_final = 0 return flex.double(list(lh.x)), (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final)
def optimize(self, I_r_flex, observations_original, wavelength, crystal_init_orientation, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams, pres_in, observations_non_polar, detector_distance_mm): ph = partiality_handler() lph = lbfgs_partiality_handler() if iparams.postref.allparams.flag_on: refine_steps = ['allparams'] else: refine_steps = ['crystal_orientation'] if iparams.postref.reflecting_range.flag_on: refine_steps.append('reflecting_range') if iparams.postref.unit_cell.flag_on: refine_steps.append('unit_cell') #get miller array iso, if given. miller_array_iso = None #prepare data pr_d_min = iparams.postref.allparams.d_min pr_d_max = iparams.postref.allparams.d_max pr_sigma_min = iparams.postref.allparams.sigma_min pr_partiality_min = iparams.postref.allparams.partiality_min pr_uc_tol = iparams.postref.allparams.uc_tolerance cs = observations_original.crystal_symmetry().space_group().crystal_system() #filter by resolution observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, I_r_flex) #filter by sigma observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel) #initialize values only in the first sub cycle and the first refine step. spot_radius = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()), observations_original_sel.indices(), wavelength) if pres_in is None: ry, rz, r0, re, voigt_nu, rotx, roty = 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, 0.0, 0.0 #apply constrain on the unit cell using crystal system uc_scale_inp = lph.prep_input(observations_original.unit_cell().parameters(), cs) uc_scale_constrained = lph.prep_output(uc_scale_inp, cs) a,b,c,alpha,beta,gamma = uc_scale_constrained const_params_scale = (rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma) xopt_scalefactors, stats = self.optimize_scalefactors(I_r_flex, observations_original, wavelength, crystal_init_orientation, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams, pres_in, observations_non_polar, detector_distance_mm, const_params_scale) G, B = xopt_scalefactors else: G, B, ry, rz, r0, re, voigt_nu, rotx, roty = pres_in.G, pres_in.B, pres_in.ry, pres_in.rz, pres_in.r0, pres_in.re, pres_in.voigt_nu, 0.0 , 0.0 a,b,c,alpha,beta,gamma = pres_in.unit_cell.parameters() crystal_init_orientation = pres_in.crystal_orientation #filter by partiality two_theta = observations_original_sel.two_theta(wavelength=wavelength).data() uc = unit_cell((a,b,c,alpha,beta,gamma)) partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty, observations_original_sel.indices(), ry, rz, r0, re, voigt_nu, two_theta, alpha_angle_sel, wavelength, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'partiality', [pr_partiality_min], observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel, partiality_in=partiality_init) I_r_true = I_ref_sel[:] I_o_true = observations_original_sel.data()[:] #calculate initial residual_xy error const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu) xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs) args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams) uc_params_err = lph.func(xinp_uc, args_uc) init_residual_xy_err = flex.sum(uc_params_err**2) #calculate initial residual_pr error const_params_all= (G,B) xinp_all = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu]) xinp_all.extend(lph.prep_input((a,b,c,alpha,beta,gamma), cs)) args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams) all_params_err = lph.func(xinp_all, args_all) init_residual_err = flex.sum(all_params_err**2) #keep in list t_pr_list = [init_residual_err] t_xy_list = [init_residual_xy_err] refined_params_hist = [(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)] txt_out = '' for i_sub_cycle in range(iparams.n_postref_sub_cycle): for j_refine_step in range(len(refine_steps)): refine_mode = refine_steps[j_refine_step] #prepare data init_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma) observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.prepare_data_microcycle(refine_mode, iparams, observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, I_r_flex, init_params, crystal_init_orientation, wavelength, detector_distance_mm) I_r_true = I_ref_sel[:] I_o_true = observations_original_sel.data() if refine_mode == 'crystal_orientation': xinp = flex.double([rotx, roty]) const_params = (G, B, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma) elif refine_mode == 'reflecting_range': xinp = flex.double([ry, rz, r0, re, voigt_nu]) const_params = (G, B, rotx, roty, a, b, c, alpha, beta, gamma) elif refine_mode == 'unit_cell': xinp = lph.prep_input((a,b,c,alpha,beta,gamma), cs) const_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu) elif refine_mode == 'allparams': xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu]) xinp.extend(lph.prep_input((a,b,c,alpha,beta,gamma), cs)) const_params = (G,B) args=(I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, refine_mode, const_params, B, miller_array_iso, iparams) lh = lbfgs_handler(current_x=xinp, args=args) xopt = flex.double(list(lh.x)) if refine_mode == 'crystal_orientation' or \ refine_mode == 'reflecting_range' or refine_mode == 'allparams': current_residual_err = lh.f #calculate residual_xy_error (for refine_mode = SF, CO, RR, and all params) xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs) if refine_mode == 'crystal_orientation': rotx, roty = xopt elif refine_mode == 'reflecting_range': ry, rz, r0, re, voigt_nu = xopt elif refine_mode == 'allparams': rotx, roty, ry, rz, r0, re, voigt_nu = xopt[:7] xinp_uc = xopt[7:] a, b, c, alpha, beta, gamma = lph.prep_output(xinp_uc, cs) const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu) xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs) args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams) uc_params_err = lph.func(xinp_uc, args_uc) current_residual_xy_err = flex.sum(uc_params_err**2) elif refine_mode == 'unit_cell': current_residual_xy_err = lh.f xopt_uc = lph.prep_output(xopt, cs) a, b, c, alpha, beta, gamma = xopt_uc #check the unit-cell with the reference intensity xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu]) xinp.extend(lph.prep_input((a, b, c, alpha, beta, gamma), cs)) const_params_all = (G,B) args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams) all_params_err = lph.func(xinp_all, args_all) current_residual_err = flex.sum(all_params_err**2) flag_success = False if refine_mode == 'allparams': #if allparams refinement, only check the post-refine target function if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \ (t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)): t_pr_list.append(current_residual_err) t_xy_list.append(current_residual_xy_err) refined_params_hist.append((G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)) flag_success = True else: if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \ (t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)): if current_residual_xy_err < (t_xy_list[len(t_xy_list)-1] + \ (t_xy_list[len(t_xy_list)-1]*iparams.postref.residual_threshold_xy/100)): t_pr_list.append(current_residual_err) t_xy_list.append(current_residual_xy_err) refined_params_hist.append((G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)) flag_success = True if flag_success is False: G,B,rotx,roty,ry,rz,r0,re,voigt_nu,a,b,c,alpha,beta,gamma = refined_params_hist[len(refined_params_hist)-1] tmp_txt_out = refine_mode + ' %3.0f %6.4f %6.4f %6.4f %6.4f %10.8f %10.8f %10.8f %10.8f %10.8f %6.3f %6.3f %.4g %6.3f\n'%(i_sub_cycle,G,B,rotx*180/math.pi,roty*180/math.pi,ry,rz,r0,re,voigt_nu,a,c,t_pr_list[len(t_pr_list)-1],t_xy_list[len(t_pr_list)-1]) txt_out += tmp_txt_out #apply the refined parameters on the full (original) reflection set two_theta = observations_original.two_theta(wavelength=wavelength).data() sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data() if pres_in is None: partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\ observations_original.unit_cell(),0.0, 0.0,observations_original.indices(), 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, two_theta, alpha_angle, wavelength, crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm, iparams.partiality_model,iparams.flag_beam_divergence) I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, 1, 0, partiality_init, rs_init) else: partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\ pres_in.unit_cell,0.0, 0.0,observations_original.indices(), pres_in.ry, pres_in.rz,pres_in.r0, pres_in.re, pres_in.voigt_nu, two_theta, alpha_angle, wavelength, crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm, iparams.partiality_model,iparams.flag_beam_divergence) I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, pres_in.G, pres_in.B, partiality_init, rs_init) partiality_fin, delta_xy_fin, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(\ unit_cell((a,b,c,alpha,beta,gamma)),rotx, roty,observations_original.indices(), ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength,crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm, iparams.partiality_model,iparams.flag_beam_divergence) I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, G, B, partiality_fin, rs_fin) SE_of_the_estimate = standard_error_of_the_estimate(I_r_flex,I_o_fin, 13) R_sq = coefficient_of_determination(I_r_flex,I_o_fin)*100 CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient() CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient() err_init = (I_r_flex - I_o_init)/observations_original.sigmas() R_init = math.sqrt(flex.sum(err_init**2)) err_final = (I_r_flex - I_o_fin)/observations_original.sigmas() R_final = math.sqrt(flex.sum(err_final**2)) R_xy_init = math.sqrt(flex.sum(delta_xy_init**2)) R_xy_final = math.sqrt(flex.sum(delta_xy_fin**2)) if R_init < R_final or re > (iparams.gamma_e * 10): CC_final = CC_init R_final = R_init R_xy_final = R_xy_init if pres_in is None: G,B,r0,ry,rz,re,rotx,roty = (1.0,0.0,spot_radius,0.0,0.0,iparams.gamma_e,0.0,0.0) a,b,c,alpha,beta,gamma = observations_original.unit_cell().parameters() else: G,B,r0,ry,rz,re,rotx,roty = (pres_in.G,pres_in.B,pres_in.r0,pres_in.ry,pres_in.rz,pres_in.re,0.0,0.0) a,b,c,alpha,beta,gamma = pres_in.unit_cell.parameters() crystal_init_orientation = pres_in.crystal_orientation #calculate CCiso if hklisoin is given CC_iso_init,CC_iso_final = (0,0) if iparams.hklisoin is not None: if miller_array_iso is not None: from cctbx import miller matches = miller.match_multi_indices( miller_indices_unique=miller_array_iso.indices(), miller_indices=observations_non_polar.indices()) I_iso_match = flex.double([miller_array_iso.data()[pair[0]] for pair in matches.pairs()]) I_o_init_match = flex.double([I_o_init[pair[1]] for pair in matches.pairs()]) I_o_fin_match = flex.double([I_o_fin[pair[1]] for pair in matches.pairs()]) CC_iso_init = flex.linear_correlation(I_iso_match, I_o_init_match).coefficient() CC_iso_final = flex.linear_correlation(I_iso_match, I_o_fin_match).coefficient() xopt = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,b,c,alpha,beta,gamma) return xopt, (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final), len(I_ref_sel)
def prepare_data_microcycle(self, refine_mode, iparams, observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, I_r_flex, init_params, crystal_init_orientation, wavelength, detector_distance_mm): #prepare data if refine_mode == 'crystal_orientation': pr_d_min = iparams.postref.crystal_orientation.d_min pr_d_max = iparams.postref.crystal_orientation.d_max pr_sigma_min = iparams.postref.crystal_orientation.sigma_min pr_partiality_min = iparams.postref.crystal_orientation.partiality_min pr_uc_tol = iparams.postref.unit_cell.uc_tolerance elif refine_mode == 'reflecting_range': pr_d_min = iparams.postref.reflecting_range.d_min pr_d_max = iparams.postref.reflecting_range.d_max pr_sigma_min = iparams.postref.reflecting_range.sigma_min pr_partiality_min = iparams.postref.reflecting_range.partiality_min pr_uc_tol = iparams.postref.unit_cell.uc_tolerance elif refine_mode == 'unit_cell': pr_d_min = iparams.postref.unit_cell.d_min pr_d_max = iparams.postref.unit_cell.d_max pr_sigma_min = iparams.postref.unit_cell.sigma_min pr_partiality_min = iparams.postref.unit_cell.partiality_min pr_uc_tol = iparams.postref.unit_cell.uc_tolerance elif refine_mode == 'allparams': pr_d_min = iparams.postref.allparams.d_min pr_d_max = iparams.postref.allparams.d_max pr_sigma_min = iparams.postref.allparams.sigma_min pr_partiality_min = iparams.postref.allparams.partiality_min pr_uc_tol = iparams.postref.unit_cell.uc_tolerance #filter by resolution observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, I_r_flex) #filter by sigma observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel) #extract refined parameters G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = init_params #filter by partiality two_theta = observations_original_sel.two_theta(wavelength=wavelength).data() uc = unit_cell((a,b,c,alpha,beta,gamma)) ph = partiality_handler() partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty, observations_original_sel.indices(), ry, rz, r0, re, voigt_nu, two_theta, alpha_angle_sel, wavelength, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'partiality', [pr_partiality_min], observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel, partiality_in=partiality_init) return observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel
def scale_frame_by_mean_I(self, frame_no, pickle_filename, iparams, mean_of_mean_I, avg_mode): observations_pickle = read_frame(pickle_filename) pickle_filepaths = pickle_filename.split('/') img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1] inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) txt_exception = ' {0:40} ==> '.format(img_filename_only) if inputs is not None: observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs else: txt_exception += txt_organize_input + '\n' return None, txt_exception #select only reflections matched with scale input params. #filter by resolution i_sel_res = observations_original.resolution_filter_selection( d_min=iparams.scale.d_min, d_max=iparams.scale.d_max) observations_original_sel = observations_original.select(i_sel_res) alpha_angle_sel = alpha_angle.select(i_sel_res) spot_pred_x_mm_sel = spot_pred_x_mm.select(i_sel_res) spot_pred_y_mm_sel = spot_pred_y_mm.select(i_sel_res) #filter by sigma i_sel_sigmas = ( observations_original_sel.data() / observations_original_sel.sigmas()) > iparams.scale.sigma_min observations_original_sel = observations_original_sel.select( i_sel_sigmas) alpha_angle_sel = alpha_angle_sel.select(i_sel_sigmas) spot_pred_x_mm_sel = spot_pred_x_mm_sel.select(i_sel_sigmas) spot_pred_y_mm_sel = spot_pred_y_mm_sel.select(i_sel_sigmas) observations_non_polar_sel, index_basis_name = self.get_observations_non_polar( observations_original_sel, pickle_filename, iparams) observations_non_polar, index_basis_name = self.get_observations_non_polar( observations_original, pickle_filename, iparams) uc_params = observations_original.unit_cell().parameters() ph = partiality_handler() r0 = ph.calc_spot_radius( sqr(crystal_init_orientation.reciprocal_matrix()), observations_original_sel.indices(), wavelength) #calculate first G (G, B) = (1, 0) stats = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0) if mean_of_mean_I > 0: G = flex.median(observations_original_sel.data()) / mean_of_mean_I if iparams.flag_apply_b_by_frame: try: mxh = mx_handler() asu_contents = mxh.get_asu_contents(iparams.n_residues) observations_as_f = observations_non_polar_sel.as_amplitude_array( ) binner_template_asu = observations_as_f.setup_binner( auto_binning=True) wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True) G = wp.wilson_intensity_scale_factor * 1e2 B = wp.wilson_b except Exception: txt_exception += 'warning B-factor calculation failed.\n' return None, txt_exception two_theta = observations_original.two_theta( wavelength=wavelength).data() sin_theta_over_lambda_sq = observations_original.two_theta( wavelength=wavelength).sin_theta_over_lambda_sq().data() ry, rz, re, voigt_nu, rotx, roty = (0, 0, iparams.gamma_e, iparams.voigt_nu, 0, 0) partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\ crystal_init_orientation.unit_cell(), rotx, roty, observations_original.indices(), ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) if iparams.flag_plot_expert: n_bins = 20 binner = observations_original.setup_binner(n_bins=n_bins) binner_indices = binner.bin_indices() avg_partiality_init = flex.double() avg_rs_init = flex.double() avg_rh_init = flex.double() one_dsqr_bin = flex.double() for i in range(1, n_bins + 1): i_binner = (binner_indices == i) if len(observations_original.data().select(i_binner)) > 0: print binner.bin_d_range(i)[1], flex.mean( partiality_init.select(i_binner)), flex.mean( rs_init.select(i_binner)), flex.mean( rh_init.select(i_binner)), len( partiality_init.select(i_binner)) #monte-carlo merge if iparams.flag_monte_carlo: G = 1 B = 0 partiality_init = flex.double([1] * len(partiality_init)) #save results refined_params = flex.double([ G, B, rotx, roty, ry, rz, r0, re, voigt_nu, uc_params[0], uc_params[1], uc_params[2], uc_params[3], uc_params[4], uc_params[5] ]) pres = postref_results() pres.set_params(observations=observations_non_polar, observations_original=observations_original, refined_params=refined_params, stats=stats, partiality=partiality_init, rs_set=rs_init, rh_set=rh_init, frame_no=frame_no, pickle_filename=pickle_filename, wavelength=wavelength, crystal_orientation=crystal_init_orientation, detector_distance_mm=detector_distance_mm) txt_scale_frame_by_mean_I = ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} G:{3:6.4f} B:{4:6.1f} CELL:{5:6.2f} {6:6.2f} {7:6.2f} {8:6.2f} {9:6.2f} {10:6.2f}'.format( img_filename_only + ' (' + index_basis_name + ')', observations_original.d_min(), len(observations_original_sel.data()), G, B, uc_params[0], uc_params[1], uc_params[2], uc_params[3], uc_params[4], uc_params[5]) print txt_scale_frame_by_mean_I txt_scale_frame_by_mean_I += '\n' return pres, txt_scale_frame_by_mean_I
def set_params(self, observations=None, observations_original=None, G_lowres=None, postref_params=None, partiality=None, rh_set=None, rs_set=None, stats=None, bragg_angle_set=None, alpha_angle_set=None, spot_pred_x_mm_set=None, spot_pred_y_mm_set=None, wavelength=None, crystal_orientation=None, detector_distance_mm=None, pickle_filename=None): """ reset observations and postrefinement parameters """ if observations is not None: self.observations = observations.deep_copy() if observations_original is not None: self.observations_original = observations_original.deep_copy() if G_lowres is not None: self.G_lowres = G_lowres if postref_params is not None: self.postref_params = postref_params[:] self.G, self.b11, self.b22, self.b33, self.b12, self.b13, self.b23, self.rotx, self.roty, self.ry, self.rz, self.r0, self.re = postref_params[: 13] self.uc_params = flex.double(postref_params[13:]) self.unit_cell = unit_cell( (self.uc_params[0], self.uc_params[1], self.uc_params[2], self.uc_params[3], self.uc_params[4], self.uc_params[5])) if partiality is not None: self.partiality = partiality[:] if rh_set is not None: self.rh_set = rh_set[:] if rs_set is not None: self.rs_set = rs_set[:] if stats is not None: self.stats = stats[:] self.cc_init = stats[0] self.cc_final = stats[1] self.r_init = stats[2] self.r_final = stats[3] self.r_xy_init = stats[4] self.r_xy_final = stats[5] self.cc_iso_init = stats[6] self.cc_iso_final = stats[7] if self.r_init == 0: self.r_change = 0 else: self.r_change = ( (self.r_final - self.r_init) / self.r_init) * 100 if self.cc_init == 0: self.cc_change = 0 else: self.cc_change = ( (self.cc_final - self.cc_init) / self.cc_init) * 100 if bragg_angle_set is not None: self.bragg_angle_set = bragg_angle_set[:] if alpha_angle_set is not None: self.alpha_angle_set = alpha_angle_set[:] if spot_pred_x_mm_set is not None: self.spot_pred_x_mm_set = spot_pred_x_mm_set[:] if spot_pred_y_mm_set is not None: self.spot_pred_y_mm_set = spot_pred_y_mm_set[:] if wavelength is not None: self.wavelength = wavelength if crystal_orientation is not None: self.crystal_orientation = crystal_orientation if detector_distance_mm is not None: self.detector_distance_mm = detector_distance_mm if pickle_filename is not None: self.pickle_filename = pickle_filename pickle_filepaths = pickle_filename.split('/') if len(pickle_filepaths) > 0: self.pickle_filename_only = pickle_filepaths[ len(pickle_filepaths) - 1] else: self.pickle_filename_only = pickle_filename #set partiality if self.postref_params is not None and self.partiality is None and self.crystal_orientation is not None: partiality_hdlr = partiality_handler(self.iparams) self.partiality, dum0, self.rh_set, self.rs_set = partiality_hdlr.calc_partiality_anisotropy_set( self) #get asu_contents self.asu_contents = {} asu_volume = self.observations.unit_cell().volume() / float( self.observations.space_group().order_z()) number_carbons = asu_volume / 18.0 self.asu_contents.setdefault('C', number_carbons)
def determine_polar(self, observations_original, iparams, pickle_filename, pres=None): """ Determine polarity based on input data. The function still needs isomorphous reference so, if flag_polar is True, miller_array_iso must be supplied in input file. """ if iparams.indexing_ambiguity.flag_on == False: return 'h,k,l', 0 , 0 cc_asu = 0 cc_rev = 0 if iparams.indexing_ambiguity.index_basis_in is not None: if iparams.indexing_ambiguity.index_basis_in.endswith('mtz'): #use reference mtz file to determine polarity from iotbx import reflection_file_reader reflection_file_polar = reflection_file_reader.any_reflection_file(iparams.indexing_ambiguity.index_basis_in) miller_arrays_polar=reflection_file_polar.as_miller_arrays() miller_array_polar = miller_arrays_polar[0] miller_array_polar = miller_array_polar.resolution_filter(d_min=iparams.indexing_ambiguity.d_min, d_max=iparams.indexing_ambiguity.d_max) #for post-refinement, apply the scale factors and partiality first if pres is not None: #observations_original = pres.observations_original.deep_copy() two_theta = observations_original.two_theta(wavelength=pres.wavelength).data() alpha_angle = flex.double([0]*len(observations_original.indices())) spot_pred_x_mm = flex.double([0]*len(observations_original.indices())) spot_pred_y_mm = flex.double([0]*len(observations_original.indices())) detector_distance_mm = pres.detector_distance_mm ph = partiality_handler() partiality, dummy, dummy, dummy = ph.calc_partiality_anisotropy_set(pres.unit_cell, 0, 0, observations_original.indices(), pres.ry, pres.rz, pres.r0, pres.re, two_theta, alpha_angle, pres.wavelength, pres.crystal_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) #partiality = pres.partiality sin_theta_over_lambda_sq = observations_original.two_theta(pres.wavelength).sin_theta_over_lambda_sq().data() I_full = flex.double(observations_original.data()/(pres.G * flex.exp(flex.double(-2*pres.B*sin_theta_over_lambda_sq)) * partiality)) sigI_full = flex.double(observations_original.sigmas()/(pres.G * flex.exp(flex.double(-2*pres.B*sin_theta_over_lambda_sq)) * partiality)) observations_original = observations_original.customized_copy(data=I_full, sigmas=sigI_full) observations_asu = observations_original.map_to_asu() observations_rev = self.get_observations_non_polar(observations_original, iparams.indexing_ambiguity.assigned_basis) matches = miller.match_multi_indices( miller_indices_unique=miller_array_polar.indices(), miller_indices=observations_asu.indices()) I_ref_match = flex.double([miller_array_polar.data()[pair[0]] for pair in matches.pairs()]) I_obs_match = flex.double([observations_asu.data()[pair[1]] for pair in matches.pairs()]) cc_asu = flex.linear_correlation(I_ref_match, I_obs_match).coefficient() n_refl_asu = len(matches.pairs()) matches = miller.match_multi_indices( miller_indices_unique=miller_array_polar.indices(), miller_indices=observations_rev.indices()) I_ref_match = flex.double([miller_array_polar.data()[pair[0]] for pair in matches.pairs()]) I_obs_match = flex.double([observations_rev.data()[pair[1]] for pair in matches.pairs()]) cc_rev = flex.linear_correlation(I_ref_match, I_obs_match).coefficient() n_refl_rev = len(matches.pairs()) polar_hkl = 'h,k,l' if cc_rev > (cc_asu*1.01): polar_hkl = iparams.indexing_ambiguity.assigned_basis else: #use basis in the given input file polar_hkl = 'h,k,l' basis_pickle = pickle.load(open(iparams.indexing_ambiguity.index_basis_in,"rb")) if pickle_filename in basis_pickle: polar_hkl = basis_pickle[pickle_filename] else: #set default polar_hkl to h,k,l polar_hkl = 'h,k,l' return polar_hkl, cc_asu, cc_rev
def func(self, params, args): I_r = args[0] miller_array_o = args[1] wavelength = args[2] alpha_angle_set = args[3] crystal_init_orientation = args[4] spot_pred_x_mm_set = args[5] spot_pred_y_mm_set = args[6] detector_distance_mm = args[7] refine_mode = args[8] const_params = args[9] b0 = args[10] miller_array_iso = args[11] iparams = args[12] partiality_model = iparams.partiality_model flag_volume_correction = iparams.flag_volume_correction flag_beam_divergence = iparams.flag_beam_divergence b_refine_d_min = iparams.b_refine_d_min I_o = miller_array_o.data() sigI_o = miller_array_o.sigmas() miller_indices_original = miller_array_o.indices() sin_theta_over_lambda_sq = miller_array_o.two_theta( wavelength=wavelength).sin_theta_over_lambda_sq().data() two_theta_flex = miller_array_o.two_theta(wavelength=wavelength).data() cs = miller_array_o.crystal_symmetry().space_group().crystal_system() if refine_mode == 'scale_factor': G, B = params rotx, roty, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = const_params elif refine_mode == 'crystal_orientation': rotx, roty = params G, B, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = const_params elif refine_mode == 'reflecting_range': ry, rz, r0, re, nu = params G, B, rotx, roty, a, b, c, alpha, beta, gamma = const_params elif refine_mode == 'unit_cell': a, b, c, alpha, beta, gamma = self.prep_output(params, cs) G, B, rotx, roty, ry, rz, r0, re, nu = const_params elif refine_mode == 'allparams': a, b, c, alpha, beta, gamma = self.prep_output(params[7:], cs) rotx, roty, ry, rz, r0, re, nu = params[0:7] G, B = const_params try: uc = unit_cell((a, b, c, alpha, beta, gamma)) except Exception: return None G, B, rotx, roty, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = flex.double( [ G, B, rotx, roty, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma ]) ph = partiality_handler() p_calc_flex, delta_xy_flex, rs_set, dummy = ph.calc_partiality_anisotropy_set(\ uc, rotx, roty, miller_indices_original, ry, rz, r0, re, nu, two_theta_flex, alpha_angle_set, wavelength, crystal_init_orientation, spot_pred_x_mm_set, spot_pred_y_mm_set, detector_distance_mm, partiality_model, flag_beam_divergence) if miller_array_o.d_min() < b_refine_d_min: I_o_full = ph.calc_full_refl(I_o, sin_theta_over_lambda_sq, G, B, p_calc_flex, rs_set, flag_volume_correction) else: I_o_full = ph.calc_full_refl(I_o, sin_theta_over_lambda_sq, G, b0, p_calc_flex, rs_set, flag_volume_correction) if refine_mode == 'unit_cell': error = delta_xy_flex else: error = ((I_r - I_o_full) / sigI_o) """ import math print refine_mode, 'G:%.4g B:%.4g rotx:%.4g roty:%.4g r0:%.4g re:%.4g nu:%6.4f a:%.4g b:%.4g c:%.4g fpr:%.4g fxy:%.4g n_refl=%5.0f'%(G, B, rotx*180/math.pi, roty*180/math.pi, r0, re, nu, a, b, c, flex.sum(((I_r - I_o_full)/sigI_o)**2), flex.sum(delta_xy_flex**2), len(I_o_full)) """ return error
def postrefine_by_frame(self, frame_no, pres_in, iparams, miller_array_ref, avg_mode): #Prepare data if pres_in is None: return None, 'Found empty pickle file' observations_pickle = pickle.load(open(pres_in.pickle_filename,"rb")) wavelength = observations_pickle["wavelength"] crystal_init_orientation = observations_pickle["current_orientation"][0] pickle_filename = pres_in.pickle_filename pickle_filepaths = pickle_filename.split('/') img_filename_only = pickle_filepaths[len(pickle_filepaths)-1] txt_exception = ' {0:40} ==> '.format(img_filename_only) inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) if inputs is not None: observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \ detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs else: txt_exception += txt_organize_input + '\n' return None, txt_exception #Select data for post-refinement (only select indices that are common with the reference set observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams) matches = miller.match_multi_indices( miller_indices_unique=miller_array_ref.indices(), miller_indices=observations_non_polar.indices()) I_ref_match = flex.double([miller_array_ref.data()[pair[0]] for pair in matches.pairs()]) miller_indices_ref_match = flex.miller_index((miller_array_ref.indices()[pair[0]] for pair in matches.pairs())) I_obs_match = flex.double([observations_non_polar.data()[pair[1]] for pair in matches.pairs()]) sigI_obs_match = flex.double([observations_non_polar.sigmas()[pair[1]] for pair in matches.pairs()]) miller_indices_original_obs_match = flex.miller_index((observations_original.indices()[pair[1]] \ for pair in matches.pairs())) miller_indices_non_polar_obs_match = flex.miller_index((observations_non_polar.indices()[pair[1]] \ for pair in matches.pairs())) alpha_angle_set = flex.double([alpha_angle[pair[1]] for pair in matches.pairs()]) spot_pred_x_mm_set = flex.double([spot_pred_x_mm[pair[1]] for pair in matches.pairs()]) spot_pred_y_mm_set = flex.double([spot_pred_y_mm[pair[1]] for pair in matches.pairs()]) references_sel = miller_array_ref.customized_copy(data=I_ref_match, indices=miller_indices_ref_match) observations_original_sel = observations_original.customized_copy(data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_original_obs_match) observations_non_polar_sel = observations_non_polar.customized_copy(data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_non_polar_obs_match) #Do least-squares refinement lsqrh = leastsqr_handler() try: refined_params, stats, n_refl_postrefined = lsqrh.optimize(I_ref_match, observations_original_sel, wavelength, crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set, spot_pred_y_mm_set, iparams, pres_in, observations_non_polar_sel, detector_distance_mm) except Exception: txt_exception += 'optimization failed.\n' return None, txt_exception #caculate partiality for output (with target_anomalous check) G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \ a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \ detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams) from cctbx.uctbx import unit_cell uc_fin = unit_cell((a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin)) crystal_init_orientation = pres_in.crystal_orientation two_theta = observations_original.two_theta(wavelength=wavelength).data() ph = partiality_handler() partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(uc_fin, rotx_fin, roty_fin, observations_original.indices(), ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) #calculate the new crystal orientation O = sqr(uc_fin.orthogonalization_matrix()).transpose() R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type CO = crystal_orientation(O*R, basis_type.direct) crystal_fin_orientation = CO.rotate_thru((1,0,0), rotx_fin ).rotate_thru((0,1,0), roty_fin) #remove reflections with partiality below threshold i_sel = partiality_fin > iparams.merge.partiality_min partiality_fin_sel = partiality_fin.select(i_sel) rs_fin_sel = rs_fin.select(i_sel) rh_fin_sel = rh_fin.select(i_sel) observations_non_polar_sel = observations_non_polar.select(i_sel) observations_original_sel = observations_original.select(i_sel) mapped_predictions = mapped_predictions.select(i_sel) pres = postref_results() pres.set_params(observations = observations_non_polar_sel, observations_original = observations_original_sel, refined_params=refined_params, stats=stats, partiality=partiality_fin_sel, rs_set=rs_fin_sel, rh_set=rh_fin_sel, frame_no=frame_no, pickle_filename=pickle_filename, wavelength=wavelength, crystal_orientation=crystal_init_orientation, detector_distance_mm=detector_distance_mm, identified_isoform=identified_isoform, mapped_predictions=mapped_predictions, xbeam=xbeam, ybeam=ybeam) r_change, r_xy_change, cc_change, cc_iso_change = (0,0,0,0) try: r_change = ((pres.R_final - pres.R_init)/pres.R_init)*100 r_xy_change = ((pres.R_xy_final - pres.R_xy_init)/pres.R_xy_init)*100 cc_change = ((pres.CC_final - pres.CC_init)/pres.CC_init)*100 cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init)/pres.CC_iso_init)*100 except Exception: pass txt_postref= ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}'.format(img_filename_only+' ('+index_basis_name+')', observations_original_sel.d_min(), len(observations_original_sel.data()), r_change, r_xy_change, cc_change, cc_iso_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin) print txt_postref txt_postref += '\n' return pres, txt_postref
def jacobian_callable(pfh,current_values): rotx = current_values[0] roty = current_values[1] ry = spot_radius rz = spot_radius G = scale_factors[0] B_factor = scale_factors[1] I_obs = observations_original_sel.data() sigI_obs = observations_original_sel.sigmas() observations_original_sel_two_theta = observations_original_sel.two_theta(wavelength=wavelength) two_theta = observations_original_sel_two_theta.data() sin_theta_over_lambda_sq = observations_original_sel_two_theta.sin_theta_over_lambda_sq().data() delta_rot = 0.0001*math.pi/180 delta_r = 0.00000001 #0. Calculate current partiality based on current rotx crystal_current_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx ).rotate_thru((0,1,0),roty) a_star = sqr(crystal_current_orientation.reciprocal_matrix()) ph = partiality_handler(wavelength, 0) #1. Calculate partial derivatives of J function Ai = sqr(crystal_init_orientation.reciprocal_matrix()) Rx = col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(rotx) Ry = col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(roty) Rz = col((0,0,1)).axis_and_angle_as_r3_rotation_matrix(0.0) dRx_drotx = col((1,0,0)).axis_and_angle_as_r3_derivative_wrt_angle(rotx) dRy_droty = col((0,1,0)).axis_and_angle_as_r3_derivative_wrt_angle(roty) dA_drotx = Rz * Ry * dRx_drotx * Ai dA_droty = Rz * dRy_droty * Rx * Ai s0 = -1*col((0,0,1./wavelength)) pDg_pDrotx = flex.double() pDg_pDroty = flex.double() pDg_pDry = flex.double() pDg_pDrz = flex.double() for j in range(len(observations_original_sel.indices())): miller_index = observations_original_sel.indices()[j] hvec = col(miller_index) xvec = a_star * hvec svec = s0 + xvec p, rh, rs = ph.calc_partiality_anisotropy(a_star, miller_index, ry, rz, two_theta[j]) #for rotx pDxvec_pDrotx = dA_drotx * hvec pDrh_pDrotx = (svec.dot(pDxvec_pDrotx))/svec.length() #for roty pDxvec_pDroty = dA_droty * hvec pDrh_pDroty = (svec.dot(pDxvec_pDroty))/svec.length() #shared derivatives on rotx and roty pDP_pDrh = (-4*(rs**2)*rh)/((2*(rh**2))+(rs**2))**2 pDg_pDP = (-1 * G * (math.exp(-2*B_factor*sin_theta_over_lambda_sq[j])) * I_obs[j])/(sigI_obs[j]*(p**2)) pDg_pDrotx.append(pDg_pDP * pDP_pDrh * pDrh_pDrotx) pDg_pDroty.append(pDg_pDP * pDP_pDrh * pDrh_pDroty) """ #checking partial derivatives g_now = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx,roty,ry,rz)) g_delta_rotx = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx+delta_rot,roty,ry,rz)) g_delta_roty = self.compute_cost_refine_crystal(I_ref, observations_original_sel, crystal_init_orientation, wavelength, (G,B_factor,rotx,roty+delta_rot,ry,rz)) pDg_pDrotx_fd = (g_delta_rotx - g_now)/delta_rot pDg_pDroty_fd = (g_delta_roty - g_now)/delta_rot print sum(flex.abs(pDg_pDrotx_fd)), sum(flex.abs(pDg_pDrotx_fd-pDg_pDrotx)) print sum(flex.abs(pDg_pDroty_fd)), sum(flex.abs(pDg_pDroty_fd-pDg_pDroty)) print """ return pDg_pDrotx, pDg_pDroty
def gradient_descend(self,I_ref, I_obs, sigI_obs, crystal_init_orientation, miller_indices_ori, wavelength, parameters, alpha, n_iters): m = len(I_ref) G = parameters[0] rotx = parameters[1] roty = parameters[2] rotz = parameters[3] rs = parameters[4] delta_rot = 0.00001*math.pi/180 J_history = flex.double([0]*n_iters) converge_iter = n_iters - 1 for i in range(n_iters): #0. Calculate current partiality based on current rotation crystal_current_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx ).rotate_thru((0,1,0),roty ).rotate_thru((0,0,1),rotz) a_star = sqr(crystal_current_orientation.reciprocal_matrix()) ph = partiality_handler(wavelength, rs) partiality = flex.double([ph.calc_partiality(a_star, miller_index)[0] for miller_index in miller_indices_ori]) #1. Calculate partial derivatives of J function Ai = sqr(crystal_init_orientation.reciprocal_matrix()) Rx = col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(rotx) Ry = col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(roty) Rz = col((0,0,1)).axis_and_angle_as_r3_rotation_matrix(rotz) dRx_drotx = col((1,0,0)).axis_and_angle_as_r3_derivative_wrt_angle(rotx) dRy_droty = col((0,1,0)).axis_and_angle_as_r3_derivative_wrt_angle(roty) dRz_drotz = col((0,0,1)).axis_and_angle_as_r3_derivative_wrt_angle(rotz) dA_drotx = Rz * Ry * dRx_drotx * Ai dA_droty = Rz * dRy_droty * Rx * Ai dA_drotz = dRz_drotz * Ry * Rx * Ai s0 = -1*col((0,0,1./wavelength)) pDJ_pDrotx = 0 pDJ_pDroty = 0 pDJ_pDrotz = 0 pDJ_pDG = 0 pDJ_pDrs = 0 for j in range(len(miller_indices_ori)): miller_index = miller_indices_ori[j] hvec = col(miller_index) xvec = a_star * hvec svec = s0 + xvec p, rh = ph.calc_partiality(a_star, miller_index) #for rotx pDxvec_pDrotx = dA_drotx * hvec pDrh_pDrotx = (svec.dot(pDxvec_pDrotx))/svec.length() """ #for finite differences #Rx_deltax = col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(rotx+delta_rot) #a_star_deltax = Ry * Rx_deltax * Ai crystal_deltax_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx+delta_rot ).rotate_thru((0,1,0),roty ).rotate_thru((0,0,1),rotz) a_star_deltax = sqr(crystal_deltax_orientation.reciprocal_matrix()) p_deltax, rh_deltax = ph.calc_partiality(a_star_deltax, miller_index) xvec_deltax = a_star_deltax * hvec svec_deltax = s0 + xvec_deltax pDxvec_pDrotx_fd = (xvec_deltax - xvec)/delta_rot pDrh_pDrotx_fd = (rh_deltax - rh)/delta_rot if j==1: print pDrh_pDrotx, pDrh_pDrotx_fd, pDrh_pDrotx-pDrh_pDrotx_fd """ #for roty pDxvec_pDroty = dA_droty * hvec pDrh_pDroty = (svec.dot(pDxvec_pDroty))/svec.length() """ #for finite differences crystal_deltay_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx ).rotate_thru((0,1,0),roty+delta_rot ).rotate_thru((0,0,1),rotz) a_star_deltay = sqr(crystal_deltay_orientation.reciprocal_matrix()) p_deltay, rh_deltay = ph.calc_partiality(a_star_deltay, miller_index) xvec_deltay = a_star_deltay * hvec svec_deltay = s0 + xvec_deltay pDxvec_pDroty_fd = (xvec_deltay - xvec)/delta_rot pDrh_pDroty_fd = (rh_deltay - rh)/delta_rot if j==1: print pDrh_pDroty, pDrh_pDroty_fd, pDrh_pDroty-pDrh_pDroty_fd """ #for rotz pDxvec_pDrotz = dA_drotz * hvec pDrh_pDrotz = (svec.dot(pDxvec_pDrotz))/svec.length() #for finite differences crystal_deltaz_orientation = crystal_init_orientation.rotate_thru((1,0,0),rotx ).rotate_thru((0,1,0),roty ).rotate_thru((0,0,1),rotz+delta_rot) a_star_deltaz = sqr(crystal_deltaz_orientation.reciprocal_matrix()) p_deltaz, rh_deltaz = ph.calc_partiality(a_star_deltaz, miller_index) xvec_deltaz = a_star_deltaz * hvec svec_deltaz = s0 + xvec_deltaz pDxvec_pDrotz = (xvec_deltaz - xvec)/delta_rot pDrh_pDrotz = (rh_deltaz - rh)/delta_rot #shared derivatives on rotx and roty pDP_pDrh = (-4*(rs**2)*rh)/((2*(rh**2))+(rs**2))**2 pDg_pDP = (-1 * G * I_obs[j])/(sigI_obs[j]*(p**2)) #for G pDg_pDG = I_obs[j] / (sigI_obs[j] * p) #for rs pDP_pDrs = (4*rs*(rh**2))/(((2*(rh**2))+(rs**2))**2) pDJ = ((((G * I_obs[j])/p) - I_ref[j])/sigI_obs[j])/m pDJ_pDrotx += pDJ * pDg_pDP * pDP_pDrh * pDrh_pDrotx pDJ_pDroty += pDJ * pDg_pDP * pDP_pDrh * pDrh_pDroty pDJ_pDrotz += pDJ * pDg_pDP * pDP_pDrh * pDrh_pDrotz pDJ_pDG += pDJ * pDg_pDG pDJ_pDrs += pDJ * pDg_pDP * pDP_pDrs """ #finite differences for PDJ #Calculate finite differences of J function over rotation angle x and y J_at_rotx_and_delta = compute_cost(I_ref, I_obs, crystal_init_orientation, miller_indices, ph, (k, G, rotx+delta_rot, roty)) J_at_roty_and_delta = compute_cost(I_ref, I_obs, crystal_init_orientation, miller_indices, ph, (k, G, rotx, roty+delta_rot)) J_at_rot = compute_cost(I_ref, I_obs, crystal_init_orientation, miller_indices, ph, (k, G, rotx, roty)) pDJ_pDrotx_fd = (J_at_rotx_and_delta - J_at_rot)/delta_rot pDJ_pDroty_fd = (J_at_roty_and_delta - J_at_rot)/delta_rot print i, pDJ_pDrotx, pDJ_pDrotx_fd, pDJ_pDroty, pDJ_pDroty_fd """ #update parameters rotx = rotx - (alpha*pDJ_pDrotx) roty = roty - (alpha*pDJ_pDroty) rotz = rotz - (alpha*pDJ_pDrotz) G = G - (alpha*pDJ_pDG) rs = rs - (alpha*pDJ_pDrs) J_history[i] = self.compute_cost(I_ref, I_obs, sigI_obs, crystal_init_orientation, miller_indices_ori, wavelength, (G, rotx, roty, rotz, rs)) print i, G, rotx*180/math.pi, roty*180/math.pi, rotz*180/math.pi, rs #check convergence gradient_threshold = 0.05 if i > 0: if abs((J_history[i]-J_history[i-1])/delta_rot) < gradient_threshold: converge_iter = i break return G, rotx, roty, rotz, rs
def optimize(self, I_r_flex, observations_original, wavelength, crystal_init_orientation, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams, pres_in, observations_non_polar, detector_distance_mm): ph = partiality_handler() lph = lbfgs_partiality_handler() if iparams.postref.allparams.flag_on: refine_steps = ['allparams'] else: refine_steps = ['crystal_orientation'] if iparams.postref.reflecting_range.flag_on: refine_steps.append('reflecting_range') if iparams.postref.unit_cell.flag_on: refine_steps.append('unit_cell') #get miller array iso, if given. miller_array_iso = None #prepare data pr_d_min = iparams.postref.allparams.d_min pr_d_max = iparams.postref.allparams.d_max pr_sigma_min = iparams.postref.allparams.sigma_min pr_partiality_min = iparams.postref.allparams.partiality_min pr_uc_tol = iparams.postref.allparams.uc_tolerance cs = observations_original.crystal_symmetry().space_group( ).crystal_system() #filter by resolution observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, I_r_flex) #filter by sigma observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel) #initialize values only in the first sub cycle and the first refine step. spot_radius = ph.calc_spot_radius( sqr(crystal_init_orientation.reciprocal_matrix()), observations_original_sel.indices(), wavelength) if pres_in is None: ry, rz, r0, re, voigt_nu, rotx, roty = 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, 0.0, 0.0 #apply constrain on the unit cell using crystal system uc_scale_inp = lph.prep_input( observations_original.unit_cell().parameters(), cs) uc_scale_constrained = lph.prep_output(uc_scale_inp, cs) a, b, c, alpha, beta, gamma = uc_scale_constrained const_params_scale = (rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma) xopt_scalefactors, stats = self.optimize_scalefactors( I_r_flex, observations_original, wavelength, crystal_init_orientation, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams, pres_in, observations_non_polar, detector_distance_mm, const_params_scale) G, B = xopt_scalefactors else: G, B, ry, rz, r0, re, voigt_nu, rotx, roty = pres_in.G, pres_in.B, pres_in.ry, pres_in.rz, pres_in.r0, pres_in.re, pres_in.voigt_nu, 0.0, 0.0 a, b, c, alpha, beta, gamma = pres_in.unit_cell.parameters() crystal_init_orientation = pres_in.crystal_orientation #filter by partiality two_theta = observations_original_sel.two_theta( wavelength=wavelength).data() uc = unit_cell((a, b, c, alpha, beta, gamma)) partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set( uc, rotx, roty, observations_original_sel.indices(), ry, rz, r0, re, voigt_nu, two_theta, alpha_angle_sel, wavelength, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'partiality', [pr_partiality_min], observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel, partiality_in=partiality_init) I_r_true = I_ref_sel[:] I_o_true = observations_original_sel.data()[:] #calculate initial residual_xy error const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu) xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs) args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams) uc_params_err = lph.func(xinp_uc, args_uc) init_residual_xy_err = flex.sum(uc_params_err**2) #calculate initial residual_pr error const_params_all = (G, B) xinp_all = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu]) xinp_all.extend(lph.prep_input((a, b, c, alpha, beta, gamma), cs)) args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams) all_params_err = lph.func(xinp_all, args_all) init_residual_err = flex.sum(all_params_err**2) #keep in list t_pr_list = [init_residual_err] t_xy_list = [init_residual_xy_err] refined_params_hist = [(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)] txt_out = '' for i_sub_cycle in range(iparams.n_postref_sub_cycle): for j_refine_step in range(len(refine_steps)): refine_mode = refine_steps[j_refine_step] #prepare data init_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma) observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.prepare_data_microcycle(refine_mode, iparams, observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, I_r_flex, init_params, crystal_init_orientation, wavelength, detector_distance_mm) I_r_true = I_ref_sel[:] I_o_true = observations_original_sel.data() if refine_mode == 'crystal_orientation': xinp = flex.double([rotx, roty]) const_params = (G, B, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma) elif refine_mode == 'reflecting_range': xinp = flex.double([ry, rz, r0, re, voigt_nu]) const_params = (G, B, rotx, roty, a, b, c, alpha, beta, gamma) elif refine_mode == 'unit_cell': xinp = lph.prep_input((a, b, c, alpha, beta, gamma), cs) const_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu) elif refine_mode == 'allparams': xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu]) xinp.extend( lph.prep_input((a, b, c, alpha, beta, gamma), cs)) const_params = (G, B) args = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, refine_mode, const_params, B, miller_array_iso, iparams) lh = lbfgs_handler(current_x=xinp, args=args) xopt = flex.double(list(lh.x)) if refine_mode == 'crystal_orientation' or \ refine_mode == 'reflecting_range' or refine_mode == 'allparams': current_residual_err = lh.f #calculate residual_xy_error (for refine_mode = SF, CO, RR, and all params) xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs) if refine_mode == 'crystal_orientation': rotx, roty = xopt elif refine_mode == 'reflecting_range': ry, rz, r0, re, voigt_nu = xopt elif refine_mode == 'allparams': rotx, roty, ry, rz, r0, re, voigt_nu = xopt[:7] xinp_uc = xopt[7:] a, b, c, alpha, beta, gamma = lph.prep_output( xinp_uc, cs) const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu) xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs) args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams) uc_params_err = lph.func(xinp_uc, args_uc) current_residual_xy_err = flex.sum(uc_params_err**2) elif refine_mode == 'unit_cell': current_residual_xy_err = lh.f xopt_uc = lph.prep_output(xopt, cs) a, b, c, alpha, beta, gamma = xopt_uc #check the unit-cell with the reference intensity xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu]) xinp.extend( lph.prep_input((a, b, c, alpha, beta, gamma), cs)) const_params_all = (G, B) args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams) all_params_err = lph.func(xinp_all, args_all) current_residual_err = flex.sum(all_params_err**2) flag_success = False if refine_mode == 'allparams': #if allparams refinement, only check the post-refine target function if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \ (t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)): t_pr_list.append(current_residual_err) t_xy_list.append(current_residual_xy_err) refined_params_hist.append( (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)) flag_success = True else: if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \ (t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)): if current_residual_xy_err < (t_xy_list[len(t_xy_list)-1] + \ (t_xy_list[len(t_xy_list)-1]*iparams.postref.residual_threshold_xy/100)): t_pr_list.append(current_residual_err) t_xy_list.append(current_residual_xy_err) refined_params_hist.append( (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)) flag_success = True if flag_success is False: G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = refined_params_hist[ len(refined_params_hist) - 1] tmp_txt_out = refine_mode + ' %3.0f %6.4f %6.4f %6.4f %6.4f %10.8f %10.8f %10.8f %10.8f %10.8f %6.3f %6.3f %.4g %6.3f\n' % ( i_sub_cycle, G, B, rotx * 180 / math.pi, roty * 180 / math.pi, ry, rz, r0, re, voigt_nu, a, c, t_pr_list[ len(t_pr_list) - 1], t_xy_list[len(t_pr_list) - 1]) txt_out += tmp_txt_out #apply the refined parameters on the full (original) reflection set two_theta = observations_original.two_theta( wavelength=wavelength).data() sin_theta_over_lambda_sq = observations_original.two_theta( wavelength=wavelength).sin_theta_over_lambda_sq().data() if pres_in is None: partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\ observations_original.unit_cell(),0.0, 0.0,observations_original.indices(), 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, two_theta, alpha_angle, wavelength, crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm, iparams.partiality_model,iparams.flag_beam_divergence) I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, 1, 0, partiality_init, rs_init) else: partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\ pres_in.unit_cell,0.0, 0.0,observations_original.indices(), pres_in.ry, pres_in.rz,pres_in.r0, pres_in.re, pres_in.voigt_nu, two_theta, alpha_angle, wavelength, crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm, iparams.partiality_model,iparams.flag_beam_divergence) I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, pres_in.G, pres_in.B, partiality_init, rs_init) partiality_fin, delta_xy_fin, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(\ unit_cell((a,b,c,alpha,beta,gamma)),rotx, roty,observations_original.indices(), ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength,crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm, iparams.partiality_model,iparams.flag_beam_divergence) I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, G, B, partiality_fin, rs_fin) SE_of_the_estimate = standard_error_of_the_estimate( I_r_flex, I_o_fin, 13) R_sq = coefficient_of_determination(I_r_flex, I_o_fin) * 100 CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient() CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient() err_init = (I_r_flex - I_o_init) / observations_original.sigmas() R_init = math.sqrt(flex.sum(err_init**2)) err_final = (I_r_flex - I_o_fin) / observations_original.sigmas() R_final = math.sqrt(flex.sum(err_final**2)) R_xy_init = math.sqrt(flex.sum(delta_xy_init**2)) R_xy_final = math.sqrt(flex.sum(delta_xy_fin**2)) if R_init < R_final or re > (iparams.gamma_e * 3): CC_final = CC_init R_final = R_init R_xy_final = R_xy_init if pres_in is None: G, B, r0, ry, rz, re, rotx, roty = (1.0, 0.0, spot_radius, 0.0, 0.0, iparams.gamma_e, 0.0, 0.0) a, b, c, alpha, beta, gamma = observations_original.unit_cell( ).parameters() else: G, B, r0, ry, rz, re, rotx, roty = (pres_in.G, pres_in.B, pres_in.r0, pres_in.ry, pres_in.rz, pres_in.re, 0.0, 0.0) a, b, c, alpha, beta, gamma = pres_in.unit_cell.parameters() crystal_init_orientation = pres_in.crystal_orientation #calculate CCiso if hklisoin is given CC_iso_init, CC_iso_final = (0, 0) if iparams.hklisoin is not None: if miller_array_iso is not None: from cctbx import miller matches = miller.match_multi_indices( miller_indices_unique=miller_array_iso.indices(), miller_indices=observations_non_polar.indices()) I_iso_match = flex.double([ miller_array_iso.data()[pair[0]] for pair in matches.pairs() ]) I_o_init_match = flex.double( [I_o_init[pair[1]] for pair in matches.pairs()]) I_o_fin_match = flex.double( [I_o_fin[pair[1]] for pair in matches.pairs()]) CC_iso_init = flex.linear_correlation( I_iso_match, I_o_init_match).coefficient() CC_iso_final = flex.linear_correlation( I_iso_match, I_o_fin_match).coefficient() xopt = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma) return xopt, (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final), len(I_ref_sel)
def optimize_scalefactors(self, I_r_flex, observations_original, wavelength, crystal_init_orientation, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams, pres_in, observations_non_polar, detector_distance_mm, const_params): ph = partiality_handler() pr_d_min = iparams.postref.scale.d_min pr_d_max = iparams.postref.scale.d_max pr_sigma_min = iparams.postref.scale.sigma_min pr_partiality_min = iparams.postref.scale.partiality_min #filter by resolution observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,\ spot_pred_x_mm, spot_pred_y_mm, I_r_flex) #filter by sigma observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \ spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\ 'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,\ spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel) I_r_true = I_ref_sel[:] I_o_true = observations_original_sel.data()[:] if pres_in is not None: G, B, b0 = pres_in.G, pres_in.B, pres_in.B else: G, B, b0 = (1, 0, 0) refine_mode = 'scale_factor' xinp = flex.double([G, B]) args = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel, crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel, detector_distance_mm, refine_mode, const_params, b0, None, iparams) lh = lbfgs_handler(current_x=xinp, args=args) G_fin, B_fin = (lh.x[0], lh.x[1]) rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = const_params two_theta = observations_original.two_theta(wavelength=wavelength) sin_theta_over_lambda_sq = two_theta.sin_theta_over_lambda_sq().data() uc = unit_cell((a, b, c, alpha, beta, gamma)) ph = partiality_handler() partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set( uc, rotx, roty, observations_original.indices(), ry, rz, r0, re, voigt_nu, two_theta.data(), alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, G, B, partiality_init, rs_init) I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq, G_fin, B_fin, partiality_init, rs_init) SE_of_the_estimate = standard_error_of_the_estimate( I_r_flex, I_o_fin, 2) R_sq = coefficient_of_determination(I_r_flex, I_o_fin) * 100 CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient() CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient() err_init = (I_r_flex - I_o_init) / observations_original.sigmas() R_init = math.sqrt(flex.sum(err_init**2)) err_final = (I_r_flex - I_o_fin) / observations_original.sigmas() R_final = math.sqrt(flex.sum(err_final**2)) R_xy_init = 0 R_xy_final = 0 CC_iso_init = 0 CC_iso_final = 0 return flex.double(list(lh.x)), (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final)
def optimize(self, I_r_flex, observations_original, wavelength, crystal_init_orientation, alpha_angle_set, iph): assert len(alpha_angle_set)==len(observations_original.indices()), 'Size of alpha angles and observations are not equal %6.0f, %6.0f'%(len(alpha_angle_set),len(observations_original.indices())) uc_init = crystal_init_orientation.unit_cell() uc_init_params = uc_init.parameters() I_r_true = I_r_flex.as_numpy_array() I_o_true = observations_original.data().as_numpy_array() sigI_o_true = observations_original.sigmas().as_numpy_array() sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data().as_numpy_array() two_theta_flex = observations_original.two_theta(wavelength=wavelength).data() cs = observations_original.crystal_symmetry().space_group().crystal_system() #calculate spot_radius a_star_true = sqr(crystal_init_orientation.reciprocal_matrix()) spot_radius = calc_spot_radius(a_star_true, observations_original.indices(), wavelength) ph = partiality_handler(wavelength, spot_radius) #1. first optain best G and k from linregress x0 = np.array([1, 0]) xopt_scale, success = optimize.leastsq(func_scale, x0, args=(I_r_true, observations_original, wavelength)) #2. optimize rot, rs, uc x0_all = np.array([xopt_scale[0], xopt_scale[1], 0, 0, spot_radius, spot_radius, 0.0026, uc_init_params[0], uc_init_params[1],uc_init_params[2], uc_init_params[3],uc_init_params[4], uc_init_params[5]]) x0 = prep_input(x0_all, cs) try: xopt_limit, cov_x, infodict, errmsg, success = optimize.leastsq(func, x0, args=(I_r_true, observations_original, ph, crystal_init_orientation, alpha_angle_set, iph), full_output=True) except Exception: xopt_limit = x0 cov_x = None xopt = prep_output(xopt_limit, cs) G, B, rotx, roty, ry, rz, re, a, b, c, alpha, beta, gamma = xopt #3. decide wheter to take the refined parameters if (abs(a-iph.target_unit_cell[0]) > iph.uc_len_tol or abs(b-iph.target_unit_cell[1]) > iph.uc_len_tol or abs(c-iph.target_unit_cell[2]) > iph.uc_len_tol \ or abs(alpha-iph.target_unit_cell[3]) > iph.uc_angle_tol or abs(beta-iph.target_unit_cell[4]) > iph.uc_angle_tol or abs(gamma-iph.target_unit_cell[5]) > iph.uc_angle_tol): if iph.flag_force_accept_all_frames: a, b, c, alpha, beta, gamma = uc_init_params rotx = 0 roty = 0 ry = spot_radius rz = spot_radius re = 0.0026 G, B = xopt_scale xopt = np.array([G, B, rotx, roty, ry, rz, re, a, b, c, alpha, beta, gamma]) #caclculate stats uc_opt = unit_cell((a,b,c,alpha,beta,gamma)) crystal_orientation_opt = get_crystal_orientation(uc_opt.orthogonalization_matrix(), crystal_init_orientation.crystal_rotation_matrix()) crystal_orientation_model = crystal_orientation_opt.rotate_thru((1,0,0), rotx ).rotate_thru((0,1,0), roty) a_star_model = sqr(crystal_orientation_model.reciprocal_matrix()) partiality_model_flex = ph.calc_partiality_anisotropy_set(a_star_model, observations_original.indices(), ry, rz, re, two_theta_flex, alpha_angle_set) partiality_model = partiality_model_flex.as_numpy_array() I_o_model = G * np.exp(-2*B*sin_theta_over_lambda_sq) * partiality_model * I_r_true SE_of_the_estimate = standard_error_of_the_estimate(I_o_true/sigI_o_true, I_o_model/sigI_o_true, len(x0)) R_sq = coefficient_of_determination(I_o_true/sigI_o_true, I_o_model/sigI_o_true)*100 CC = np.corrcoef(I_o_true/sigI_o_true, I_o_model/sigI_o_true)[0,1] var_I_p = ((observations_original.sigmas()/observations_original.data())**2).as_numpy_array() #calculate standard error for the parameters if cov_x is None: se_xopt = np.array([0,0,0,0,0,0,0,0,0,0,0,0,0]) var_k = flex.double([0]*len(I_o_model)) var_p = flex.double([0]*len(I_o_model)) SE_I = np.sqrt(var_I_p)*(I_o_model) else: pcov = cov_x * (SE_of_the_estimate**2) var_xopt_limit = np.array([pcov[i,i] for i in range(len(x0))]) var_xopt = prep_variance(var_xopt_limit, cs) se_xopt = np.sqrt(var_xopt) #calculate error propagation in I_full se_G, se_B, se_rotx, se_roty, se_ry, se_rz, se_re, se_a, se_b, se_c, se_alpha, se_beta, se_gamma = se_xopt ###k = G_0 * exp(-2*B*sin_theta_over_lambda_sq) se_k_sq = (np.exp(-2*B*sin_theta_over_lambda_sq)*se_G)**2 + (-2 * G * sin_theta_over_lambda_sq * np.exp(-2*B*sin_theta_over_lambda_sq) * se_B)**2 var_k = se_k_sq/((G*np.exp(-2*B*sin_theta_over_lambda_sq))**2) ###use finite differences to propagate errors in partility function p. var_p_sq = flex.double() fmode_arr = ('G','B','rotx','roty','ry','rz','re','a','b','c','alpha','beta','gamma') for miller_index, bragg_angle, alpha_angle in zip(observations_original.indices(), two_theta_flex, alpha_angle_set): Dp = flex.double() cn_fmode = 0 for fmode in fmode_arr: dp = misc.derivative(func_partiality, xopt[cn_fmode], args=(miller_index, crystal_init_orientation.crystal_rotation_matrix(), wavelength, (bragg_angle, alpha_angle), xopt, fmode)) cn_fmode += 1 Dp.append(dp) se_p_sq = 0 for dp, se in zip(Dp, se_xopt): se_p_sq += (dp*se)**2 var_p_sq.append(se_p_sq) var_p = (var_p_sq/(partiality_model_flex**2)).as_numpy_array() SE_I = np.sqrt(var_I_p + var_k + var_p)*(I_o_model) """ print 'Predictor' print 'G %.4g'%(G) print 'B-factor %.4g'%(B) print 'rotx %.4g'%(rotx*180/math.pi) print 'roty %.4g'%(roty*180/math.pi) print 'ry %.4g'%(ry) print 'rz %.4g'%(rz) print 're %.4g'%(re) print 'uc', uc_opt print 'S = %.4g'%SE_of_the_estimate print 'R-Sq = %.4g%%'%(R_sq) print 'CC = %.4g'%(CC) plt.scatter(I_r_true, I_o_true,s=10, marker='x', c='r') plt.scatter(I_r_true, I_o_model,s=10, marker='o', c='b') plt.title('G=%.4g B=%.4g rotx=%.4g roty=%.4g R-sq=%.4g%%'%(xopt[0], xopt[1], xopt[2]*180/math.pi, xopt[3]*180/math.pi, R_sq)) plt.xlabel('I_ref') plt.ylabel('I_obs') plt.show() """ return xopt, se_xopt, (SE_of_the_estimate, R_sq, CC), partiality_model, SE_I, var_I_p, var_k, var_p
def postrefine_by_frame(self, frame_no, pickle_filename, iparams, miller_array_ref, pres_in, avg_mode): #1. Prepare data observations_pickle = read_frame(pickle_filename) pickle_filepaths = pickle_filename.split('/') img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1] txt_exception = ' {0:40} ==> '.format(img_filename_only) inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) if inputs is not None: observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs else: txt_exception += txt_organize_input + '\n' return None, txt_exception #2. Select data for post-refinement (only select indices that are common with the reference set observations_non_polar, index_basis_name = self.get_observations_non_polar( observations_original, pickle_filename, iparams) matches = miller.match_multi_indices( miller_indices_unique=miller_array_ref.indices(), miller_indices=observations_non_polar.indices()) pair_0 = flex.size_t([pair[0] for pair in matches.pairs()]) pair_1 = flex.size_t([pair[1] for pair in matches.pairs()]) references_sel = miller_array_ref.select(pair_0) observations_original_sel = observations_original.select(pair_1) observations_non_polar_sel = observations_non_polar.select(pair_1) alpha_angle_set = alpha_angle.select(pair_1) spot_pred_x_mm_set = spot_pred_x_mm.select(pair_1) spot_pred_y_mm_set = spot_pred_y_mm.select(pair_1) #4. Do least-squares refinement lsqrh = leastsqr_handler() try: refined_params, stats, n_refl_postrefined = lsqrh.optimize( references_sel.data(), observations_original_sel, wavelength, crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set, spot_pred_y_mm_set, iparams, pres_in, observations_non_polar_sel, detector_distance_mm) except Exception: txt_exception += 'optimization failed.\n' return None, txt_exception #caculate partiality for output (with target_anomalous check) G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \ a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs observations_non_polar, index_basis_name = self.get_observations_non_polar( observations_original, pickle_filename, iparams) from cctbx.uctbx import unit_cell uc_fin = unit_cell( (a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin)) if pres_in is not None: crystal_init_orientation = pres_in.crystal_orientation two_theta = observations_original.two_theta( wavelength=wavelength).data() ph = partiality_handler() partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set( uc_fin, rotx_fin, roty_fin, observations_original.indices(), ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) #calculate the new crystal orientation O = sqr(uc_fin.orthogonalization_matrix()).transpose() R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type CO = crystal_orientation(O * R, basis_type.direct) crystal_fin_orientation = CO.rotate_thru( (1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin) #remove reflections with partiality below threshold i_sel = partiality_fin > iparams.merge.partiality_min partiality_fin_sel = partiality_fin.select(i_sel) rs_fin_sel = rs_fin.select(i_sel) rh_fin_sel = rh_fin.select(i_sel) observations_non_polar_sel = observations_non_polar.customized_copy(\ indices=observations_non_polar.indices().select(i_sel), data=observations_non_polar.data().select(i_sel), sigmas=observations_non_polar.sigmas().select(i_sel)) observations_original_sel = observations_original.customized_copy(\ indices=observations_original.indices().select(i_sel), data=observations_original.data().select(i_sel), sigmas=observations_original.sigmas().select(i_sel)) pres = postref_results() pres.set_params(observations=observations_non_polar_sel, observations_original=observations_original_sel, refined_params=refined_params, stats=stats, partiality=partiality_fin_sel, rs_set=rs_fin_sel, rh_set=rh_fin_sel, frame_no=frame_no, pickle_filename=pickle_filename, wavelength=wavelength, crystal_orientation=crystal_fin_orientation, detector_distance_mm=detector_distance_mm) r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100 r_xy_change = ( (pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100 cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100 txt_postref = '{0:40} => RES:{1:5.2f} NREFL:{2:5d} R:{3:6.1f}% RXY:{4:5.1f}% CC:{5:5.1f}% G:{6:6.4f} B:{7:5.1f} CELL:{8:6.1f}{9:6.1f} {10:6.1f} {11:5.1f} {12:5.1f} {13:5.1f}'.format( img_filename_only + ' (' + index_basis_name + ')', observations_original_sel.d_min(), len(observations_original_sel.data()), r_change, r_xy_change, cc_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin) print txt_postref txt_postref += '\n' return pres, txt_postref
def scale_frame_by_mean_I(self, frame_no, pickle_filename, iparams, mean_of_mean_I, avg_mode='average'): observations_pickle = pickle.load(open(pickle_filename,"rb")) pickle_filepaths = pickle_filename.split('/') img_filename_only = pickle_filepaths[len(pickle_filepaths)-1] inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) txt_exception = ' {0:40} ==> '.format(img_filename_only) if inputs is not None: observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \ detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs else: txt_exception += txt_organize_input + '\n' return None, txt_exception wavelength = observations_pickle["wavelength"] crystal_init_orientation = observations_pickle["current_orientation"][0] #select only reflections matched with scale input params. #filter by resolution i_sel_res = observations_original.resolution_filter_selection(d_min=iparams.scale.d_min, d_max=iparams.scale.d_max) observations_original_sel = observations_original.select(i_sel_res) alpha_angle_sel = alpha_angle.select(i_sel_res) spot_pred_x_mm_sel = spot_pred_x_mm.select(i_sel_res) spot_pred_y_mm_sel = spot_pred_y_mm.select(i_sel_res) #filter by sigma i_sel_sigmas = (observations_original_sel.data()/observations_original_sel.sigmas()) > iparams.scale.sigma_min observations_original_sel = observations_original_sel.select(i_sel_sigmas) alpha_angle_sel = alpha_angle_sel.select(i_sel_sigmas) spot_pred_x_mm_sel = spot_pred_x_mm_sel.select(i_sel_sigmas) spot_pred_y_mm_sel = spot_pred_y_mm_sel.select(i_sel_sigmas) observations_non_polar_sel, index_basis_name = self.get_observations_non_polar(observations_original_sel, pickle_filename, iparams) observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams) uc_params = observations_original.unit_cell().parameters() ph = partiality_handler() r0 = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()), observations_original_sel.indices(), wavelength) #calculate first G (G, B) = (1,0) stats = (0,0,0,0,0,0,0,0,0,0) if mean_of_mean_I > 0: G = flex.median(observations_original_sel.data())/mean_of_mean_I if iparams.flag_apply_b_by_frame: try: from mod_util import mx_handler mxh = mx_handler() asu_contents = mxh.get_asu_contents(iparams.n_residues) observations_as_f = observations_non_polar_sel.as_amplitude_array() binner_template_asu = observations_as_f.setup_binner(auto_binning=True) wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True) G = wp.wilson_intensity_scale_factor*1e3 B = wp.wilson_b except Exception: txt_exception += 'warning B-factor calculation failed.\n' return None, txt_exception two_theta = observations_original.two_theta(wavelength=wavelength).data() sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data() ry, rz, re, voigt_nu, rotx, roty = (0, 0, iparams.gamma_e, iparams.voigt_nu, 0, 0) partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\ crystal_init_orientation.unit_cell(), rotx, roty, observations_original.indices(), ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) if iparams.flag_plot_expert: n_bins = 20 binner = observations_original.setup_binner(n_bins=n_bins) binner_indices = binner.bin_indices() avg_partiality_init = flex.double() avg_rs_init = flex.double() avg_rh_init = flex.double() one_dsqr_bin = flex.double() for i in range(1,n_bins+1): i_binner = (binner_indices == i) if len(observations_original.data().select(i_binner)) > 0: print binner.bin_d_range(i)[1], flex.mean(partiality_init.select(i_binner)), flex.mean(rs_init.select(i_binner)), flex.mean(rh_init.select(i_binner)), len(partiality_init.select(i_binner)) #save results refined_params = flex.double([G,B,rotx,roty,ry,rz,r0,re,voigt_nu,uc_params[0],uc_params[1],uc_params[2],uc_params[3],uc_params[4],uc_params[5]]) pres = postref_results() pres.set_params(observations = observations_non_polar, observations_original = observations_original, refined_params=refined_params, stats=stats, partiality=partiality_init, rs_set=rs_init, rh_set=rh_init, frame_no=frame_no, pickle_filename=pickle_filename, wavelength=wavelength, crystal_orientation=crystal_init_orientation, detector_distance_mm=detector_distance_mm, identified_isoform=identified_isoform, mapped_predictions=mapped_predictions, xbeam=xbeam, ybeam=ybeam) txt_scale_frame_by_mean_I = ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} G:{3:10.3e} B:{4:7.1f} CELL:{5:6.2f} {6:6.2f} {7:6.2f} {8:6.2f} {9:6.2f} {10:6.2f}'.format(img_filename_only+' ('+index_basis_name+')', observations_original.d_min(), len(observations_original_sel.data()), G, B, uc_params[0],uc_params[1],uc_params[2],uc_params[3],uc_params[4],uc_params[5]) print txt_scale_frame_by_mean_I txt_scale_frame_by_mean_I += '\n' return pres, txt_scale_frame_by_mean_I
def func(self, params, args): I_r = args[0] miller_array_o = args[1] wavelength = args[2] alpha_angle_set = args[3] crystal_init_orientation = args[4] spot_pred_x_mm_set = args[5] spot_pred_y_mm_set = args[6] detector_distance_mm = args[7] refine_mode = args[8] const_params = args[9] b0 = args[10] miller_array_iso = args[11] iparams = args[12] partiality_model = iparams.partiality_model flag_volume_correction = iparams.flag_volume_correction flag_beam_divergence = iparams.flag_beam_divergence b_refine_d_min = iparams.b_refine_d_min I_o = miller_array_o.data() sigI_o = miller_array_o.sigmas() miller_indices_original = miller_array_o.indices() sin_theta_over_lambda_sq = miller_array_o.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data() two_theta_flex = miller_array_o.two_theta(wavelength=wavelength).data() cs = miller_array_o.crystal_symmetry().space_group().crystal_system() if refine_mode == 'scale_factor': G, B = params rotx, roty, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = const_params elif refine_mode == 'crystal_orientation': rotx, roty = params G, B, ry, rz, r0, re, nu, a, b, c, alpha, beta, gamma = const_params elif refine_mode == 'reflecting_range': ry, rz, r0, re, nu = params G, B, rotx, roty, a, b, c, alpha, beta, gamma = const_params elif refine_mode == 'unit_cell': a, b, c, alpha, beta, gamma = self.prep_output(params, cs) G, B, rotx, roty, ry, rz, r0, re, nu = const_params elif refine_mode == 'allparams': a, b, c, alpha, beta, gamma = self.prep_output(params[7:], cs) rotx, roty, ry, rz, r0, re, nu = params[0:7] G,B = const_params try: uc = unit_cell((a,b,c,alpha,beta,gamma)) except Exception: return None G,B,rotx,roty,ry,rz,r0,re,nu,a,b,c,alpha,beta,gamma = flex.double([G,B,rotx,roty,ry,rz,r0,re,nu,a,b,c,alpha,beta,gamma]) ph = partiality_handler() p_calc_flex, delta_xy_flex, rs_set, dummy = ph.calc_partiality_anisotropy_set(\ uc, rotx, roty, miller_indices_original, ry, rz, r0, re, nu, two_theta_flex, alpha_angle_set, wavelength, crystal_init_orientation, spot_pred_x_mm_set, spot_pred_y_mm_set, detector_distance_mm, partiality_model, flag_beam_divergence) if miller_array_o.d_min() < b_refine_d_min: I_o_full = ph.calc_full_refl(I_o, sin_theta_over_lambda_sq, G, B, p_calc_flex, rs_set, flag_volume_correction) else: I_o_full = ph.calc_full_refl(I_o, sin_theta_over_lambda_sq, G, b0, p_calc_flex, rs_set, flag_volume_correction) if refine_mode == 'unit_cell': error = delta_xy_flex else: error = ((I_r - I_o_full)/sigI_o) """ import math print refine_mode, 'G:%.4g B:%.4g rotx:%.4g roty:%.4g r0:%.4g re:%.4g nu:%6.4f a:%.4g b:%.4g c:%.4g fpr:%.4g fxy:%.4g n_refl=%5.0f'%(G, B, rotx*180/math.pi, roty*180/math.pi, r0, re, nu, a, b, c, flex.sum(((I_r - I_o_full)/sigI_o)**2), flex.sum(delta_xy_flex**2), len(I_o_full)) """ return error
def postrefine_by_frame(self, frame_no, pres_in, iparams, miller_array_ref, avg_mode): #Prepare data if pres_in is None: return None, 'Found empty pickle file' observations_pickle = pickle.load(open(pres_in.pickle_filename, "rb")) wavelength = observations_pickle["wavelength"] crystal_init_orientation = observations_pickle["current_orientation"][ 0] pickle_filename = pres_in.pickle_filename pickle_filepaths = pickle_filename.split('/') img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1] txt_exception = ' {0:40} ==> '.format(img_filename_only) inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) if inputs is not None: observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \ detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs else: txt_exception += txt_organize_input + '\n' return None, txt_exception #Select data for post-refinement (only select indices that are common with the reference set observations_non_polar, index_basis_name = self.get_observations_non_polar( observations_original, pickle_filename, iparams) matches = miller.match_multi_indices( miller_indices_unique=miller_array_ref.indices(), miller_indices=observations_non_polar.indices()) I_ref_match = flex.double( [miller_array_ref.data()[pair[0]] for pair in matches.pairs()]) miller_indices_ref_match = flex.miller_index( (miller_array_ref.indices()[pair[0]] for pair in matches.pairs())) I_obs_match = flex.double([ observations_non_polar.data()[pair[1]] for pair in matches.pairs() ]) sigI_obs_match = flex.double([ observations_non_polar.sigmas()[pair[1]] for pair in matches.pairs() ]) miller_indices_original_obs_match = flex.miller_index((observations_original.indices()[pair[1]] \ for pair in matches.pairs())) miller_indices_non_polar_obs_match = flex.miller_index((observations_non_polar.indices()[pair[1]] \ for pair in matches.pairs())) alpha_angle_set = flex.double( [alpha_angle[pair[1]] for pair in matches.pairs()]) spot_pred_x_mm_set = flex.double( [spot_pred_x_mm[pair[1]] for pair in matches.pairs()]) spot_pred_y_mm_set = flex.double( [spot_pred_y_mm[pair[1]] for pair in matches.pairs()]) references_sel = miller_array_ref.customized_copy( data=I_ref_match, indices=miller_indices_ref_match) observations_original_sel = observations_original.customized_copy( data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_original_obs_match) observations_non_polar_sel = observations_non_polar.customized_copy( data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_non_polar_obs_match) #Do least-squares refinement lsqrh = leastsqr_handler() try: refined_params, stats, n_refl_postrefined = lsqrh.optimize( I_ref_match, observations_original_sel, wavelength, crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set, spot_pred_y_mm_set, iparams, pres_in, observations_non_polar_sel, detector_distance_mm) except Exception: txt_exception += 'optimization failed.\n' return None, txt_exception #caculate partiality for output (with target_anomalous check) G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \ a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \ detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs observations_non_polar, index_basis_name = self.get_observations_non_polar( observations_original, pickle_filename, iparams) from cctbx.uctbx import unit_cell uc_fin = unit_cell( (a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin)) crystal_init_orientation = pres_in.crystal_orientation two_theta = observations_original.two_theta( wavelength=wavelength).data() ph = partiality_handler() partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set( uc_fin, rotx_fin, roty_fin, observations_original.indices(), ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) #calculate the new crystal orientation O = sqr(uc_fin.orthogonalization_matrix()).transpose() R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type CO = crystal_orientation(O * R, basis_type.direct) crystal_fin_orientation = CO.rotate_thru( (1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin) #remove reflections with partiality below threshold i_sel = partiality_fin > iparams.merge.partiality_min partiality_fin_sel = partiality_fin.select(i_sel) rs_fin_sel = rs_fin.select(i_sel) rh_fin_sel = rh_fin.select(i_sel) observations_non_polar_sel = observations_non_polar.select(i_sel) observations_original_sel = observations_original.select(i_sel) mapped_predictions = mapped_predictions.select(i_sel) pres = postref_results() pres.set_params(observations=observations_non_polar_sel, observations_original=observations_original_sel, refined_params=refined_params, stats=stats, partiality=partiality_fin_sel, rs_set=rs_fin_sel, rh_set=rh_fin_sel, frame_no=frame_no, pickle_filename=pickle_filename, wavelength=wavelength, crystal_orientation=crystal_init_orientation, detector_distance_mm=detector_distance_mm, identified_isoform=identified_isoform, mapped_predictions=mapped_predictions, xbeam=xbeam, ybeam=ybeam) r_change, r_xy_change, cc_change, cc_iso_change = (0, 0, 0, 0) try: r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100 r_xy_change = ( (pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100 cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100 cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init) / pres.CC_iso_init) * 100 except Exception: pass txt_postref = ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}'.format( img_filename_only + ' (' + index_basis_name + ')', observations_original_sel.d_min(), len(observations_original_sel.data()), r_change, r_xy_change, cc_change, cc_iso_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin) print txt_postref txt_postref += '\n' return pres, txt_postref