def postrefine_by_frame(self, frame_no, pickle_filename, iparams, miller_array_ref, pres_in, avg_mode): # 1. Prepare data observations_pickle = pickle.load(open(pickle_filename, "rb")) crystal_init_orientation = observations_pickle["current_orientation"][0] wavelength = observations_pickle["wavelength"] pickle_filepaths = pickle_filename.split("/") img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1] txt_exception = " {0:40} ==> ".format(img_filename_only) inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename ) if inputs is not None: observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm = inputs else: txt_exception += txt_organize_input + "\n" return None, txt_exception # 2. Determine polarity - always do this even if flag_polar = False # the function will take care of it. polar_hkl, cc_iso_raw_asu, cc_iso_raw_rev = self.determine_polar( observations_original, iparams, pickle_filename, pres=pres_in ) # 3. Select data for post-refinement (only select indices that are common with the reference set observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl) matches = miller.match_multi_indices( miller_indices_unique=miller_array_ref.indices(), miller_indices=observations_non_polar.indices() ) I_ref_match = flex.double([miller_array_ref.data()[pair[0]] for pair in matches.pairs()]) miller_indices_ref_match = flex.miller_index((miller_array_ref.indices()[pair[0]] for pair in matches.pairs())) I_obs_match = flex.double([observations_non_polar.data()[pair[1]] for pair in matches.pairs()]) sigI_obs_match = flex.double([observations_non_polar.sigmas()[pair[1]] for pair in matches.pairs()]) miller_indices_original_obs_match = flex.miller_index( (observations_original.indices()[pair[1]] for pair in matches.pairs()) ) miller_indices_non_polar_obs_match = flex.miller_index( (observations_non_polar.indices()[pair[1]] for pair in matches.pairs()) ) alpha_angle_set = flex.double([alpha_angle[pair[1]] for pair in matches.pairs()]) spot_pred_x_mm_set = flex.double([spot_pred_x_mm[pair[1]] for pair in matches.pairs()]) spot_pred_y_mm_set = flex.double([spot_pred_y_mm[pair[1]] for pair in matches.pairs()]) references_sel = miller_array_ref.customized_copy(data=I_ref_match, indices=miller_indices_ref_match) observations_original_sel = observations_original.customized_copy( data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_original_obs_match ) observations_non_polar_sel = observations_non_polar.customized_copy( data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_non_polar_obs_match ) # 4. Do least-squares refinement lsqrh = leastsqr_handler() try: refined_params, stats, n_refl_postrefined = lsqrh.optimize( I_ref_match, observations_original_sel, wavelength, crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set, spot_pred_y_mm_set, iparams, pres_in, observations_non_polar_sel, detector_distance_mm, ) except Exception: txt_exception += "optimization failed.\n" return None, txt_exception # caculate partiality for output (with target_anomalous check) G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = ( refined_params ) inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename ) observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm = inputs observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl) from cctbx.uctbx import unit_cell uc_fin = unit_cell((a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin)) if pres_in is not None: crystal_init_orientation = pres_in.crystal_orientation two_theta = observations_original.two_theta(wavelength=wavelength).data() from mod_leastsqr import calc_partiality_anisotropy_set partiality_fin, dummy, rs_fin, rh_fin = calc_partiality_anisotropy_set( uc_fin, rotx_fin, roty_fin, observations_original.indices(), ry_fin, rz_fin, r0_fin, re_fin, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence, ) # calculate the new crystal orientation O = sqr(uc_fin.orthogonalization_matrix()).transpose() R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type CO = crystal_orientation(O * R, basis_type.direct) crystal_fin_orientation = CO.rotate_thru((1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin) # remove reflections with partiality below threshold i_sel = partiality_fin > iparams.merge.partiality_min partiality_fin_sel = partiality_fin.select(i_sel) rs_fin_sel = rs_fin.select(i_sel) rh_fin_sel = rh_fin.select(i_sel) observations_non_polar_sel = observations_non_polar.customized_copy( indices=observations_non_polar.indices().select(i_sel), data=observations_non_polar.data().select(i_sel), sigmas=observations_non_polar.sigmas().select(i_sel), ) observations_original_sel = observations_original.customized_copy( indices=observations_original.indices().select(i_sel), data=observations_original.data().select(i_sel), sigmas=observations_original.sigmas().select(i_sel), ) pres = postref_results() pres.set_params( observations=observations_non_polar_sel, observations_original=observations_original_sel, refined_params=refined_params, stats=stats, partiality=partiality_fin_sel, rs_set=rs_fin_sel, rh_set=rh_fin_sel, frame_no=frame_no, pickle_filename=pickle_filename, wavelength=wavelength, crystal_orientation=crystal_fin_orientation, detector_distance_mm=detector_distance_mm, ) r_change, r_xy_change, cc_change, cc_iso_change = (0, 0, 0, 0) try: r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100 r_xy_change = ((pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100 cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100 cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init) / pres.CC_iso_init) * 100 except Exception: pass txt_postref = " {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}".format( img_filename_only + " (" + polar_hkl + ")", observations_original_sel.d_min(), len(observations_original_sel.data()), r_change, r_xy_change, cc_change, cc_iso_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin, ) print txt_postref txt_postref += "\n" return pres, txt_postref
def scale_frame_by_mean_I(self, frame_no, pickle_filename, iparams, mean_of_mean_I, avg_mode): observations_pickle = pickle.load(open(pickle_filename, "rb")) pickle_filepaths = pickle_filename.split("/") img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1] inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename ) txt_exception = " {0:40} ==> ".format(img_filename_only) if inputs is not None: observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm = inputs else: txt_exception += txt_organize_input + "\n" return None, txt_exception wavelength = observations_pickle["wavelength"] crystal_init_orientation = observations_pickle["current_orientation"][0] # select only reflections matched with scale input params. # filter by resolution i_sel_res = observations_original.resolution_filter_selection( d_min=iparams.scale.d_min, d_max=iparams.scale.d_max ) observations_original_sel = observations_original.select(i_sel_res) alpha_angle_sel = alpha_angle.select(i_sel_res) spot_pred_x_mm_sel = spot_pred_x_mm.select(i_sel_res) spot_pred_y_mm_sel = spot_pred_y_mm.select(i_sel_res) # filter by sigma i_sel_sigmas = (observations_original_sel.data() / observations_original_sel.sigmas()) > iparams.scale.sigma_min observations_original_sel = observations_original_sel.select(i_sel_sigmas) alpha_angle_sel = alpha_angle_sel.select(i_sel_sigmas) spot_pred_x_mm_sel = spot_pred_x_mm_sel.select(i_sel_sigmas) spot_pred_y_mm_sel = spot_pred_y_mm_sel.select(i_sel_sigmas) polar_hkl, cc_iso_raw_asu, cc_iso_raw_rev = self.determine_polar( observations_original, iparams, pickle_filename ) observations_non_polar_sel = self.get_observations_non_polar(observations_original_sel, polar_hkl) observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl) uc_params = observations_original.unit_cell().parameters() from mod_leastsqr import calc_spot_radius r0 = calc_spot_radius( sqr(crystal_init_orientation.reciprocal_matrix()), observations_original_sel.indices(), wavelength ) # calculate first G (G, B) = (1, 0) stats = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0) if mean_of_mean_I > 0: G = flex.median(observations_original_sel.data()) / mean_of_mean_I if iparams.flag_apply_b_by_frame: try: from mod_util import mx_handler mxh = mx_handler() asu_contents = mxh.get_asu_contents(iparams.n_residues) observations_as_f = observations_non_polar.as_amplitude_array() binner_template_asu = observations_as_f.setup_binner(auto_binning=True) wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True) G = wp.wilson_intensity_scale_factor * 1e3 B = wp.wilson_b except Exception: txt_exception += "warning B-factor calculation failed.\n" return None, txt_exception from mod_leastsqr import calc_partiality_anisotropy_set two_theta = observations_original.two_theta(wavelength=wavelength).data() sin_theta_over_lambda_sq = ( observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data() ) ry, rz, re, rotx, roty = (0, 0, iparams.gamma_e, 0, 0) partiality_init, delta_xy_init, rs_init, rh_init = calc_partiality_anisotropy_set( crystal_init_orientation.unit_cell(), rotx, roty, observations_original.indices(), ry, rz, r0, re, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence, ) if iparams.flag_plot_expert: n_bins = 20 binner = observations_original.setup_binner(n_bins=n_bins) binner_indices = binner.bin_indices() avg_partiality_init = flex.double() avg_rs_init = flex.double() avg_rh_init = flex.double() one_dsqr_bin = flex.double() for i in range(1, n_bins + 1): i_binner = binner_indices == i if len(observations_original.data().select(i_binner)) > 0: print binner.bin_d_range(i)[1], flex.mean(partiality_init.select(i_binner)), flex.mean( rs_init.select(i_binner) ), flex.mean(rh_init.select(i_binner)), len(partiality_init.select(i_binner)) # save results refined_params = flex.double( [ G, B, rotx, roty, ry, rz, r0, re, uc_params[0], uc_params[1], uc_params[2], uc_params[3], uc_params[4], uc_params[5], ] ) pres = postref_results() pres.set_params( observations=observations_non_polar, observations_original=observations_original, refined_params=refined_params, stats=stats, partiality=partiality_init, rs_set=rs_init, rh_set=rh_init, frame_no=frame_no, pickle_filename=pickle_filename, wavelength=wavelength, crystal_orientation=crystal_init_orientation, detector_distance_mm=detector_distance_mm, ) txt_scale_frame_by_mean_I = " {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} G:{3:10.3e} B:{4:7.1f} CELL:{5:6.2f} {6:6.2f} {7:6.2f} {8:6.2f} {9:6.2f} {10:6.2f}".format( img_filename_only + " (" + polar_hkl + ")", observations_original.d_min(), len(observations_original_sel.data()), G, B, uc_params[0], uc_params[1], uc_params[2], uc_params[3], uc_params[4], uc_params[5], ) print txt_scale_frame_by_mean_I txt_scale_frame_by_mean_I += "\n" return pres, txt_scale_frame_by_mean_I
def determine_polar(self, observations_original, iparams, pickle_filename, pres=None): """ Determine polarity based on input data. The function still needs isomorphous reference so, if flag_polar is True, miller_array_iso must be supplied in input file. """ if iparams.indexing_ambiguity.flag_on == False: return "h,k,l", 0, 0 cc_asu = 0 cc_rev = 0 if iparams.indexing_ambiguity.index_basis_in is not None: if iparams.indexing_ambiguity.index_basis_in.endswith("mtz"): # use reference mtz file to determine polarity from iotbx import reflection_file_reader reflection_file_polar = reflection_file_reader.any_reflection_file( iparams.indexing_ambiguity.index_basis_in ) miller_arrays_polar = reflection_file_polar.as_miller_arrays() miller_array_polar = miller_arrays_polar[0] miller_array_polar = miller_array_polar.resolution_filter( d_min=iparams.indexing_ambiguity.d_min, d_max=iparams.indexing_ambiguity.d_max ) # for post-refinement, apply the scale factors and partiality first if pres is not None: # observations_original = pres.observations_original.deep_copy() two_theta = observations_original.two_theta(wavelength=pres.wavelength).data() from mod_leastsqr import calc_partiality_anisotropy_set alpha_angle = flex.double([0] * len(observations_original.indices())) spot_pred_x_mm = flex.double([0] * len(observations_original.indices())) spot_pred_y_mm = flex.double([0] * len(observations_original.indices())) detector_distance_mm = pres.detector_distance_mm partiality, dummy, dummy, dummy = calc_partiality_anisotropy_set( pres.unit_cell, 0, 0, observations_original.indices(), pres.ry, pres.rz, pres.r0, pres.re, two_theta, alpha_angle, pres.wavelength, pres.crystal_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence, ) # partiality = pres.partiality sin_theta_over_lambda_sq = ( observations_original.two_theta(pres.wavelength).sin_theta_over_lambda_sq().data() ) I_full = flex.double( observations_original.data() / (pres.G * flex.exp(flex.double(-2 * pres.B * sin_theta_over_lambda_sq)) * partiality) ) sigI_full = flex.double( observations_original.sigmas() / (pres.G * flex.exp(flex.double(-2 * pres.B * sin_theta_over_lambda_sq)) * partiality) ) observations_original = observations_original.customized_copy(data=I_full, sigmas=sigI_full) observations_asu = observations_original.map_to_asu() observations_rev = self.get_observations_non_polar( observations_original, iparams.indexing_ambiguity.assigned_basis ) matches = miller.match_multi_indices( miller_indices_unique=miller_array_polar.indices(), miller_indices=observations_asu.indices() ) I_ref_match = flex.double([miller_array_polar.data()[pair[0]] for pair in matches.pairs()]) I_obs_match = flex.double([observations_asu.data()[pair[1]] for pair in matches.pairs()]) cc_asu = flex.linear_correlation(I_ref_match, I_obs_match).coefficient() n_refl_asu = len(matches.pairs()) matches = miller.match_multi_indices( miller_indices_unique=miller_array_polar.indices(), miller_indices=observations_rev.indices() ) I_ref_match = flex.double([miller_array_polar.data()[pair[0]] for pair in matches.pairs()]) I_obs_match = flex.double([observations_rev.data()[pair[1]] for pair in matches.pairs()]) cc_rev = flex.linear_correlation(I_ref_match, I_obs_match).coefficient() n_refl_rev = len(matches.pairs()) polar_hkl = "h,k,l" if cc_rev > (cc_asu * 1.01): polar_hkl = iparams.indexing_ambiguity.assigned_basis else: # use basis in the given input file polar_hkl = "h,k,l" basis_pickle = pickle.load(open(iparams.indexing_ambiguity.index_basis_in, "rb")) if pickle_filename in basis_pickle: polar_hkl = basis_pickle[pickle_filename] else: # set default polar_hkl to h,k,l polar_hkl = "h,k,l" return polar_hkl, cc_asu, cc_rev