def postrefine_by_frame(self, frame_no, pickle_filename, iparams, miller_array_ref, pres_in, avg_mode):
# 1. Prepare data
observations_pickle = pickle.load(open(pickle_filename, "rb"))
crystal_init_orientation = observations_pickle["current_orientation"][0]
wavelength = observations_pickle["wavelength"]
pickle_filepaths = pickle_filename.split("/")
img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1]
txt_exception = " {0:40} ==> ".format(img_filename_only)
inputs, txt_organize_input = self.organize_input(
observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename
)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm = inputs
else:
txt_exception += txt_organize_input + "\n"
return None, txt_exception
# 2. Determine polarity - always do this even if flag_polar = False
# the function will take care of it.
polar_hkl, cc_iso_raw_asu, cc_iso_raw_rev = self.determine_polar(
observations_original, iparams, pickle_filename, pres=pres_in
)
# 3. Select data for post-refinement (only select indices that are common with the reference set
observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_ref.indices(), miller_indices=observations_non_polar.indices()
)
I_ref_match = flex.double([miller_array_ref.data()[pair[0]] for pair in matches.pairs()])
miller_indices_ref_match = flex.miller_index((miller_array_ref.indices()[pair[0]] for pair in matches.pairs()))
I_obs_match = flex.double([observations_non_polar.data()[pair[1]] for pair in matches.pairs()])
sigI_obs_match = flex.double([observations_non_polar.sigmas()[pair[1]] for pair in matches.pairs()])
miller_indices_original_obs_match = flex.miller_index(
(observations_original.indices()[pair[1]] for pair in matches.pairs())
)
miller_indices_non_polar_obs_match = flex.miller_index(
(observations_non_polar.indices()[pair[1]] for pair in matches.pairs())
)
alpha_angle_set = flex.double([alpha_angle[pair[1]] for pair in matches.pairs()])
spot_pred_x_mm_set = flex.double([spot_pred_x_mm[pair[1]] for pair in matches.pairs()])
spot_pred_y_mm_set = flex.double([spot_pred_y_mm[pair[1]] for pair in matches.pairs()])
references_sel = miller_array_ref.customized_copy(data=I_ref_match, indices=miller_indices_ref_match)
observations_original_sel = observations_original.customized_copy(
data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_original_obs_match
)
observations_non_polar_sel = observations_non_polar.customized_copy(
data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_non_polar_obs_match
)
# 4. Do least-squares refinement
lsqrh = leastsqr_handler()
try:
refined_params, stats, n_refl_postrefined = lsqrh.optimize(
I_ref_match,
observations_original_sel,
wavelength,
crystal_init_orientation,
alpha_angle_set,
spot_pred_x_mm_set,
spot_pred_y_mm_set,
iparams,
pres_in,
observations_non_polar_sel,
detector_distance_mm,
)
except Exception:
txt_exception += "optimization failed.\n"
return None, txt_exception
# caculate partiality for output (with target_anomalous check)
G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = (
refined_params
)
inputs, txt_organize_input = self.organize_input(
observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename
)
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm = inputs
observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl)
from cctbx.uctbx import unit_cell
uc_fin = unit_cell((a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin))
if pres_in is not None:
crystal_init_orientation = pres_in.crystal_orientation
two_theta = observations_original.two_theta(wavelength=wavelength).data()
from mod_leastsqr import calc_partiality_anisotropy_set
partiality_fin, dummy, rs_fin, rh_fin = calc_partiality_anisotropy_set(
uc_fin,
rotx_fin,
roty_fin,
observations_original.indices(),
ry_fin,
rz_fin,
r0_fin,
re_fin,
two_theta,
alpha_angle,
wavelength,
crystal_init_orientation,
spot_pred_x_mm,
spot_pred_y_mm,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence,
)
# calculate the new crystal orientation
O = sqr(uc_fin.orthogonalization_matrix()).transpose()
R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose()
from cctbx.crystal_orientation import crystal_orientation, basis_type
CO = crystal_orientation(O * R, basis_type.direct)
crystal_fin_orientation = CO.rotate_thru((1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin)
# remove reflections with partiality below threshold
i_sel = partiality_fin > iparams.merge.partiality_min
partiality_fin_sel = partiality_fin.select(i_sel)
rs_fin_sel = rs_fin.select(i_sel)
rh_fin_sel = rh_fin.select(i_sel)
observations_non_polar_sel = observations_non_polar.customized_copy(
indices=observations_non_polar.indices().select(i_sel),
data=observations_non_polar.data().select(i_sel),
sigmas=observations_non_polar.sigmas().select(i_sel),
)
observations_original_sel = observations_original.customized_copy(
indices=observations_original.indices().select(i_sel),
data=observations_original.data().select(i_sel),
sigmas=observations_original.sigmas().select(i_sel),
)
pres = postref_results()
pres.set_params(
observations=observations_non_polar_sel,
observations_original=observations_original_sel,
refined_params=refined_params,
stats=stats,
partiality=partiality_fin_sel,
rs_set=rs_fin_sel,
rh_set=rh_fin_sel,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_fin_orientation,
detector_distance_mm=detector_distance_mm,
)
r_change, r_xy_change, cc_change, cc_iso_change = (0, 0, 0, 0)
try:
r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100
r_xy_change = ((pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100
cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100
cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init) / pres.CC_iso_init) * 100
except Exception:
pass
txt_postref = " {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}".format(
img_filename_only + " (" + polar_hkl + ")",
observations_original_sel.d_min(),
len(observations_original_sel.data()),
r_change,
r_xy_change,
cc_change,
cc_iso_change,
pres.G,
pres.B,
a_fin,
b_fin,
c_fin,
alpha_fin,
beta_fin,
gamma_fin,
)
print txt_postref
txt_postref += "\n"
return pres, txt_postref
def scale_frame_by_mean_I(self, frame_no, pickle_filename, iparams, mean_of_mean_I, avg_mode):
observations_pickle = pickle.load(open(pickle_filename, "rb"))
pickle_filepaths = pickle_filename.split("/")
img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1]
inputs, txt_organize_input = self.organize_input(
observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename
)
txt_exception = " {0:40} ==> ".format(img_filename_only)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm = inputs
else:
txt_exception += txt_organize_input + "\n"
return None, txt_exception
wavelength = observations_pickle["wavelength"]
crystal_init_orientation = observations_pickle["current_orientation"][0]
# select only reflections matched with scale input params.
# filter by resolution
i_sel_res = observations_original.resolution_filter_selection(
d_min=iparams.scale.d_min, d_max=iparams.scale.d_max
)
observations_original_sel = observations_original.select(i_sel_res)
alpha_angle_sel = alpha_angle.select(i_sel_res)
spot_pred_x_mm_sel = spot_pred_x_mm.select(i_sel_res)
spot_pred_y_mm_sel = spot_pred_y_mm.select(i_sel_res)
# filter by sigma
i_sel_sigmas = (observations_original_sel.data() / observations_original_sel.sigmas()) > iparams.scale.sigma_min
observations_original_sel = observations_original_sel.select(i_sel_sigmas)
alpha_angle_sel = alpha_angle_sel.select(i_sel_sigmas)
spot_pred_x_mm_sel = spot_pred_x_mm_sel.select(i_sel_sigmas)
spot_pred_y_mm_sel = spot_pred_y_mm_sel.select(i_sel_sigmas)
polar_hkl, cc_iso_raw_asu, cc_iso_raw_rev = self.determine_polar(
observations_original, iparams, pickle_filename
)
observations_non_polar_sel = self.get_observations_non_polar(observations_original_sel, polar_hkl)
observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl)
uc_params = observations_original.unit_cell().parameters()
from mod_leastsqr import calc_spot_radius
r0 = calc_spot_radius(
sqr(crystal_init_orientation.reciprocal_matrix()), observations_original_sel.indices(), wavelength
)
# calculate first G
(G, B) = (1, 0)
stats = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
if mean_of_mean_I > 0:
G = flex.median(observations_original_sel.data()) / mean_of_mean_I
if iparams.flag_apply_b_by_frame:
try:
from mod_util import mx_handler
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(iparams.n_residues)
observations_as_f = observations_non_polar.as_amplitude_array()
binner_template_asu = observations_as_f.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(observations_as_f, asu_contents, e_statistics=True)
G = wp.wilson_intensity_scale_factor * 1e3
B = wp.wilson_b
except Exception:
txt_exception += "warning B-factor calculation failed.\n"
return None, txt_exception
from mod_leastsqr import calc_partiality_anisotropy_set
two_theta = observations_original.two_theta(wavelength=wavelength).data()
sin_theta_over_lambda_sq = (
observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data()
)
ry, rz, re, rotx, roty = (0, 0, iparams.gamma_e, 0, 0)
partiality_init, delta_xy_init, rs_init, rh_init = calc_partiality_anisotropy_set(
crystal_init_orientation.unit_cell(),
rotx,
roty,
observations_original.indices(),
ry,
rz,
r0,
re,
two_theta,
alpha_angle,
wavelength,
crystal_init_orientation,
spot_pred_x_mm,
spot_pred_y_mm,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence,
)
if iparams.flag_plot_expert:
n_bins = 20
binner = observations_original.setup_binner(n_bins=n_bins)
binner_indices = binner.bin_indices()
avg_partiality_init = flex.double()
avg_rs_init = flex.double()
avg_rh_init = flex.double()
one_dsqr_bin = flex.double()
for i in range(1, n_bins + 1):
i_binner = binner_indices == i
if len(observations_original.data().select(i_binner)) > 0:
print binner.bin_d_range(i)[1], flex.mean(partiality_init.select(i_binner)), flex.mean(
rs_init.select(i_binner)
), flex.mean(rh_init.select(i_binner)), len(partiality_init.select(i_binner))
# save results
refined_params = flex.double(
[
G,
B,
rotx,
roty,
ry,
rz,
r0,
re,
uc_params[0],
uc_params[1],
uc_params[2],
uc_params[3],
uc_params[4],
uc_params[5],
]
)
pres = postref_results()
pres.set_params(
observations=observations_non_polar,
observations_original=observations_original,
refined_params=refined_params,
stats=stats,
partiality=partiality_init,
rs_set=rs_init,
rh_set=rh_init,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_init_orientation,
detector_distance_mm=detector_distance_mm,
)
txt_scale_frame_by_mean_I = " {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} G:{3:10.3e} B:{4:7.1f} CELL:{5:6.2f} {6:6.2f} {7:6.2f} {8:6.2f} {9:6.2f} {10:6.2f}".format(
img_filename_only + " (" + polar_hkl + ")",
observations_original.d_min(),
len(observations_original_sel.data()),
G,
B,
uc_params[0],
uc_params[1],
uc_params[2],
uc_params[3],
uc_params[4],
uc_params[5],
)
print txt_scale_frame_by_mean_I
txt_scale_frame_by_mean_I += "\n"
return pres, txt_scale_frame_by_mean_I
def determine_polar(self, observations_original, iparams, pickle_filename, pres=None):
"""
Determine polarity based on input data.
The function still needs isomorphous reference so, if flag_polar is True,
miller_array_iso must be supplied in input file.
"""
if iparams.indexing_ambiguity.flag_on == False:
return "h,k,l", 0, 0
cc_asu = 0
cc_rev = 0
if iparams.indexing_ambiguity.index_basis_in is not None:
if iparams.indexing_ambiguity.index_basis_in.endswith("mtz"):
# use reference mtz file to determine polarity
from iotbx import reflection_file_reader
reflection_file_polar = reflection_file_reader.any_reflection_file(
iparams.indexing_ambiguity.index_basis_in
)
miller_arrays_polar = reflection_file_polar.as_miller_arrays()
miller_array_polar = miller_arrays_polar[0]
miller_array_polar = miller_array_polar.resolution_filter(
d_min=iparams.indexing_ambiguity.d_min, d_max=iparams.indexing_ambiguity.d_max
)
# for post-refinement, apply the scale factors and partiality first
if pres is not None:
# observations_original = pres.observations_original.deep_copy()
two_theta = observations_original.two_theta(wavelength=pres.wavelength).data()
from mod_leastsqr import calc_partiality_anisotropy_set
alpha_angle = flex.double([0] * len(observations_original.indices()))
spot_pred_x_mm = flex.double([0] * len(observations_original.indices()))
spot_pred_y_mm = flex.double([0] * len(observations_original.indices()))
detector_distance_mm = pres.detector_distance_mm
partiality, dummy, dummy, dummy = calc_partiality_anisotropy_set(
pres.unit_cell,
0,
0,
observations_original.indices(),
pres.ry,
pres.rz,
pres.r0,
pres.re,
two_theta,
alpha_angle,
pres.wavelength,
pres.crystal_orientation,
spot_pred_x_mm,
spot_pred_y_mm,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence,
)
# partiality = pres.partiality
sin_theta_over_lambda_sq = (
observations_original.two_theta(pres.wavelength).sin_theta_over_lambda_sq().data()
)
I_full = flex.double(
observations_original.data()
/ (pres.G * flex.exp(flex.double(-2 * pres.B * sin_theta_over_lambda_sq)) * partiality)
)
sigI_full = flex.double(
observations_original.sigmas()
/ (pres.G * flex.exp(flex.double(-2 * pres.B * sin_theta_over_lambda_sq)) * partiality)
)
observations_original = observations_original.customized_copy(data=I_full, sigmas=sigI_full)
observations_asu = observations_original.map_to_asu()
observations_rev = self.get_observations_non_polar(
observations_original, iparams.indexing_ambiguity.assigned_basis
)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_polar.indices(), miller_indices=observations_asu.indices()
)
I_ref_match = flex.double([miller_array_polar.data()[pair[0]] for pair in matches.pairs()])
I_obs_match = flex.double([observations_asu.data()[pair[1]] for pair in matches.pairs()])
cc_asu = flex.linear_correlation(I_ref_match, I_obs_match).coefficient()
n_refl_asu = len(matches.pairs())
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_polar.indices(), miller_indices=observations_rev.indices()
)
I_ref_match = flex.double([miller_array_polar.data()[pair[0]] for pair in matches.pairs()])
I_obs_match = flex.double([observations_rev.data()[pair[1]] for pair in matches.pairs()])
cc_rev = flex.linear_correlation(I_ref_match, I_obs_match).coefficient()
n_refl_rev = len(matches.pairs())
polar_hkl = "h,k,l"
if cc_rev > (cc_asu * 1.01):
polar_hkl = iparams.indexing_ambiguity.assigned_basis
else:
# use basis in the given input file
polar_hkl = "h,k,l"
basis_pickle = pickle.load(open(iparams.indexing_ambiguity.index_basis_in, "rb"))
if pickle_filename in basis_pickle:
polar_hkl = basis_pickle[pickle_filename]
else:
# set default polar_hkl to h,k,l
polar_hkl = "h,k,l"
return polar_hkl, cc_asu, cc_rev