def scale_with_reference(self,
observations_hdlr_ref,
filename=None,
observations_hdlr=None,
use_binning=False):
"""
Scale each emage to the reference.
"""
if True:
if filename is not None:
observations_hdlr = observations_handler(self.iparams)
observations_hdlr.init_params_from_file(filename)
#find G and B-tensor for high resolution reflections
lsqr_hdlr = leastsqr_handler(self.iparams)
lsqr_hdlr.optimize_scalefactors(observations_hdlr_ref,
observations_hdlr,
use_binning=use_binning)
#find G low resolution
obs_ref_lowres = observations_hdlr_ref.observations.resolution_filter(
d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
obs_lowres = observations_hdlr.observations.resolution_filter(
d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
obs_lowres = obs_lowres.select(
flex.bool(
np.absolute(
(obs_lowres.data() - np.mean(obs_lowres.data())) /
np.std(obs_lowres.data())) < self.sigma_I_cutoff))
try:
G_lowres = obs_lowres.mean() / obs_ref_lowres.mean()
except Exception:
G_lowres = observations_hdlr.G
observations_hdlr.set_params(G_lowres=G_lowres)
txt_out = ' {0:40} ==> SCALED RES:{1:5.2f} NREFL:{2:5d} SG:{14:8} Glow:{15:6.1f} G:{3:6.1f} B:{4:6.1f} {5:6.1f} {6:6.1f} {7:6.1f} {8:6.1f} {9:6.1f} Rfin:{11:5.2f} CCini:{12:5.2f} CCfin:{13:5.2f}'.format(
observations_hdlr.pickle_filename_only,
observations_hdlr.observations.d_min(),
len(observations_hdlr.observations.data()),
observations_hdlr.G, observations_hdlr.b11,
observations_hdlr.b22, observations_hdlr.b33,
observations_hdlr.b12, observations_hdlr.b13,
observations_hdlr.b23, observations_hdlr.r_init,
observations_hdlr.r_final, observations_hdlr.cc_init,
observations_hdlr.cc_final,
observations_hdlr.observations.space_group_info(), G_lowres)
#except Exception:
# return None, ' {0:40} ==> SCALING FAILED'.format(observations_hdlr.pickle_filename_only)
return observations_hdlr, txt_out
def postrefine_with_reference(self, observations_hdlr_ref,
observations_hdlr):
"""
Post-refine each image using a reference set.
"""
msg = 'POSTREF'
try:
lsqr_hdlr = leastsqr_handler(self.iparams)
lsqr_hdlr.optimize(observations_hdlr_ref, observations_hdlr)
except Exception:
msg = 'FAILED'
txt_out = ' {0:40} ==> {1:7} RES:{2:5.2f} NREFL:{3:6d} r0{4:7.4f} re{5:7.4f} CELL:{6:5.1f} {7:5.1f} {8:5.1f} {9:5.1f} {10:5.1f} {11:5.1f} Rini:{12:5.2f} Rfin:{13:5.2f} CCini:{14:5.2f} CCfin:{15:5.2f}'.format(
observations_hdlr.pickle_filename_only, msg,
observations_hdlr.observations.d_min(),
len(observations_hdlr.observations.data()), observations_hdlr.r0,
observations_hdlr.re, observations_hdlr.uc_params[0],
observations_hdlr.uc_params[1], observations_hdlr.uc_params[2],
observations_hdlr.uc_params[3], observations_hdlr.uc_params[4],
observations_hdlr.uc_params[5], observations_hdlr.r_init,
observations_hdlr.r_final, observations_hdlr.cc_init,
observations_hdlr.cc_final)
return observations_hdlr, txt_out
def postrefine_by_frame(self, frame_no, pickle_filename, iparams,
miller_array_ref, pres_in, avg_mode):
#1. Prepare data
observations_pickle = read_frame(pickle_filename)
pickle_filepaths = pickle_filename.split('/')
img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1]
txt_exception = ' {0:40} ==> '.format(img_filename_only)
inputs, txt_organize_input = self.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pickle_filename)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs
else:
txt_exception += txt_organize_input + '\n'
return None, txt_exception
#2. Select data for post-refinement (only select indices that are common with the reference set
observations_non_polar, index_basis_name = self.get_observations_non_polar(
observations_original, pickle_filename, iparams)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_ref.indices(),
miller_indices=observations_non_polar.indices())
pair_0 = flex.size_t([pair[0] for pair in matches.pairs()])
pair_1 = flex.size_t([pair[1] for pair in matches.pairs()])
references_sel = miller_array_ref.select(pair_0)
observations_original_sel = observations_original.select(pair_1)
observations_non_polar_sel = observations_non_polar.select(pair_1)
alpha_angle_set = alpha_angle.select(pair_1)
spot_pred_x_mm_set = spot_pred_x_mm.select(pair_1)
spot_pred_y_mm_set = spot_pred_y_mm.select(pair_1)
#4. Do least-squares refinement
lsqrh = leastsqr_handler()
try:
refined_params, stats, n_refl_postrefined = lsqrh.optimize(
references_sel.data(), observations_original_sel, wavelength,
crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set,
spot_pred_y_mm_set, iparams, pres_in,
observations_non_polar_sel, detector_distance_mm)
except Exception:
txt_exception += 'optimization failed.\n'
return None, txt_exception
#caculate partiality for output (with target_anomalous check)
G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \
a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params
inputs, txt_organize_input = self.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pickle_filename)
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs
observations_non_polar, index_basis_name = self.get_observations_non_polar(
observations_original, pickle_filename, iparams)
from cctbx.uctbx import unit_cell
uc_fin = unit_cell(
(a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin))
if pres_in is not None:
crystal_init_orientation = pres_in.crystal_orientation
two_theta = observations_original.two_theta(
wavelength=wavelength).data()
ph = partiality_handler()
partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(
uc_fin, rotx_fin, roty_fin, observations_original.indices(),
ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, two_theta,
alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm,
spot_pred_y_mm, detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
#calculate the new crystal orientation
O = sqr(uc_fin.orthogonalization_matrix()).transpose()
R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose()
from cctbx.crystal_orientation import crystal_orientation, basis_type
CO = crystal_orientation(O * R, basis_type.direct)
crystal_fin_orientation = CO.rotate_thru(
(1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin)
#remove reflections with partiality below threshold
i_sel = partiality_fin > iparams.merge.partiality_min
partiality_fin_sel = partiality_fin.select(i_sel)
rs_fin_sel = rs_fin.select(i_sel)
rh_fin_sel = rh_fin.select(i_sel)
observations_non_polar_sel = observations_non_polar.customized_copy(\
indices=observations_non_polar.indices().select(i_sel),
data=observations_non_polar.data().select(i_sel),
sigmas=observations_non_polar.sigmas().select(i_sel))
observations_original_sel = observations_original.customized_copy(\
indices=observations_original.indices().select(i_sel),
data=observations_original.data().select(i_sel),
sigmas=observations_original.sigmas().select(i_sel))
pres = postref_results()
pres.set_params(observations=observations_non_polar_sel,
observations_original=observations_original_sel,
refined_params=refined_params,
stats=stats,
partiality=partiality_fin_sel,
rs_set=rs_fin_sel,
rh_set=rh_fin_sel,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_fin_orientation,
detector_distance_mm=detector_distance_mm)
r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100
r_xy_change = (
(pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100
cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100
txt_postref = '{0:40} => RES:{1:5.2f} NREFL:{2:5d} R:{3:6.1f}% RXY:{4:5.1f}% CC:{5:5.1f}% G:{6:6.4f} B:{7:5.1f} CELL:{8:6.1f}{9:6.1f} {10:6.1f} {11:5.1f} {12:5.1f} {13:5.1f}'.format(
img_filename_only + ' (' + index_basis_name + ')',
observations_original_sel.d_min(),
len(observations_original_sel.data()), r_change, r_xy_change,
cc_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin,
beta_fin, gamma_fin)
print txt_postref
txt_postref += '\n'
return pres, txt_postref
def postrefine_by_frame(self, frame_no, pickle_filename, iparams, miller_array_ref, pres_in, avg_mode):
# 1. Prepare data
observations_pickle = pickle.load(open(pickle_filename, "rb"))
crystal_init_orientation = observations_pickle["current_orientation"][0]
wavelength = observations_pickle["wavelength"]
pickle_filepaths = pickle_filename.split("/")
img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1]
txt_exception = " {0:40} ==> ".format(img_filename_only)
inputs, txt_organize_input = self.organize_input(
observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename
)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm = inputs
else:
txt_exception += txt_organize_input + "\n"
return None, txt_exception
# 2. Determine polarity - always do this even if flag_polar = False
# the function will take care of it.
polar_hkl, cc_iso_raw_asu, cc_iso_raw_rev = self.determine_polar(
observations_original, iparams, pickle_filename, pres=pres_in
)
# 3. Select data for post-refinement (only select indices that are common with the reference set
observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_ref.indices(), miller_indices=observations_non_polar.indices()
)
I_ref_match = flex.double([miller_array_ref.data()[pair[0]] for pair in matches.pairs()])
miller_indices_ref_match = flex.miller_index((miller_array_ref.indices()[pair[0]] for pair in matches.pairs()))
I_obs_match = flex.double([observations_non_polar.data()[pair[1]] for pair in matches.pairs()])
sigI_obs_match = flex.double([observations_non_polar.sigmas()[pair[1]] for pair in matches.pairs()])
miller_indices_original_obs_match = flex.miller_index(
(observations_original.indices()[pair[1]] for pair in matches.pairs())
)
miller_indices_non_polar_obs_match = flex.miller_index(
(observations_non_polar.indices()[pair[1]] for pair in matches.pairs())
)
alpha_angle_set = flex.double([alpha_angle[pair[1]] for pair in matches.pairs()])
spot_pred_x_mm_set = flex.double([spot_pred_x_mm[pair[1]] for pair in matches.pairs()])
spot_pred_y_mm_set = flex.double([spot_pred_y_mm[pair[1]] for pair in matches.pairs()])
references_sel = miller_array_ref.customized_copy(data=I_ref_match, indices=miller_indices_ref_match)
observations_original_sel = observations_original.customized_copy(
data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_original_obs_match
)
observations_non_polar_sel = observations_non_polar.customized_copy(
data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_non_polar_obs_match
)
# 4. Do least-squares refinement
lsqrh = leastsqr_handler()
try:
refined_params, stats, n_refl_postrefined = lsqrh.optimize(
I_ref_match,
observations_original_sel,
wavelength,
crystal_init_orientation,
alpha_angle_set,
spot_pred_x_mm_set,
spot_pred_y_mm_set,
iparams,
pres_in,
observations_non_polar_sel,
detector_distance_mm,
)
except Exception:
txt_exception += "optimization failed.\n"
return None, txt_exception
# caculate partiality for output (with target_anomalous check)
G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = (
refined_params
)
inputs, txt_organize_input = self.organize_input(
observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename
)
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm = inputs
observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl)
from cctbx.uctbx import unit_cell
uc_fin = unit_cell((a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin))
if pres_in is not None:
crystal_init_orientation = pres_in.crystal_orientation
two_theta = observations_original.two_theta(wavelength=wavelength).data()
from mod_leastsqr import calc_partiality_anisotropy_set
partiality_fin, dummy, rs_fin, rh_fin = calc_partiality_anisotropy_set(
uc_fin,
rotx_fin,
roty_fin,
observations_original.indices(),
ry_fin,
rz_fin,
r0_fin,
re_fin,
two_theta,
alpha_angle,
wavelength,
crystal_init_orientation,
spot_pred_x_mm,
spot_pred_y_mm,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence,
)
# calculate the new crystal orientation
O = sqr(uc_fin.orthogonalization_matrix()).transpose()
R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose()
from cctbx.crystal_orientation import crystal_orientation, basis_type
CO = crystal_orientation(O * R, basis_type.direct)
crystal_fin_orientation = CO.rotate_thru((1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin)
# remove reflections with partiality below threshold
i_sel = partiality_fin > iparams.merge.partiality_min
partiality_fin_sel = partiality_fin.select(i_sel)
rs_fin_sel = rs_fin.select(i_sel)
rh_fin_sel = rh_fin.select(i_sel)
observations_non_polar_sel = observations_non_polar.customized_copy(
indices=observations_non_polar.indices().select(i_sel),
data=observations_non_polar.data().select(i_sel),
sigmas=observations_non_polar.sigmas().select(i_sel),
)
observations_original_sel = observations_original.customized_copy(
indices=observations_original.indices().select(i_sel),
data=observations_original.data().select(i_sel),
sigmas=observations_original.sigmas().select(i_sel),
)
pres = postref_results()
pres.set_params(
observations=observations_non_polar_sel,
observations_original=observations_original_sel,
refined_params=refined_params,
stats=stats,
partiality=partiality_fin_sel,
rs_set=rs_fin_sel,
rh_set=rh_fin_sel,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_fin_orientation,
detector_distance_mm=detector_distance_mm,
)
r_change, r_xy_change, cc_change, cc_iso_change = (0, 0, 0, 0)
try:
r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100
r_xy_change = ((pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100
cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100
cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init) / pres.CC_iso_init) * 100
except Exception:
pass
txt_postref = " {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}".format(
img_filename_only + " (" + polar_hkl + ")",
observations_original_sel.d_min(),
len(observations_original_sel.data()),
r_change,
r_xy_change,
cc_change,
cc_iso_change,
pres.G,
pres.B,
a_fin,
b_fin,
c_fin,
alpha_fin,
beta_fin,
gamma_fin,
)
print txt_postref
txt_postref += "\n"
return pres, txt_postref
def postrefine_by_frame(self, frame_no, pres_in, iparams, miller_array_ref, avg_mode):
#Prepare data
if pres_in is None:
return None, 'Found empty pickle file'
observations_pickle = pickle.load(open(pres_in.pickle_filename,"rb"))
wavelength = observations_pickle["wavelength"]
crystal_init_orientation = observations_pickle["current_orientation"][0]
pickle_filename = pres_in.pickle_filename
pickle_filepaths = pickle_filename.split('/')
img_filename_only = pickle_filepaths[len(pickle_filepaths)-1]
txt_exception = ' {0:40} ==> '.format(img_filename_only)
inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
else:
txt_exception += txt_organize_input + '\n'
return None, txt_exception
#Select data for post-refinement (only select indices that are common with the reference set
observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_ref.indices(),
miller_indices=observations_non_polar.indices())
I_ref_match = flex.double([miller_array_ref.data()[pair[0]] for pair in matches.pairs()])
miller_indices_ref_match = flex.miller_index((miller_array_ref.indices()[pair[0]] for pair in matches.pairs()))
I_obs_match = flex.double([observations_non_polar.data()[pair[1]] for pair in matches.pairs()])
sigI_obs_match = flex.double([observations_non_polar.sigmas()[pair[1]] for pair in matches.pairs()])
miller_indices_original_obs_match = flex.miller_index((observations_original.indices()[pair[1]] \
for pair in matches.pairs()))
miller_indices_non_polar_obs_match = flex.miller_index((observations_non_polar.indices()[pair[1]] \
for pair in matches.pairs()))
alpha_angle_set = flex.double([alpha_angle[pair[1]] for pair in matches.pairs()])
spot_pred_x_mm_set = flex.double([spot_pred_x_mm[pair[1]] for pair in matches.pairs()])
spot_pred_y_mm_set = flex.double([spot_pred_y_mm[pair[1]] for pair in matches.pairs()])
references_sel = miller_array_ref.customized_copy(data=I_ref_match, indices=miller_indices_ref_match)
observations_original_sel = observations_original.customized_copy(data=I_obs_match,
sigmas=sigI_obs_match,
indices=miller_indices_original_obs_match)
observations_non_polar_sel = observations_non_polar.customized_copy(data=I_obs_match,
sigmas=sigI_obs_match,
indices=miller_indices_non_polar_obs_match)
#Do least-squares refinement
lsqrh = leastsqr_handler()
try:
refined_params, stats, n_refl_postrefined = lsqrh.optimize(I_ref_match,
observations_original_sel, wavelength,
crystal_init_orientation, alpha_angle_set,
spot_pred_x_mm_set, spot_pred_y_mm_set,
iparams,
pres_in,
observations_non_polar_sel,
detector_distance_mm)
except Exception:
txt_exception += 'optimization failed.\n'
return None, txt_exception
#caculate partiality for output (with target_anomalous check)
G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \
a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params
inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename)
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams)
from cctbx.uctbx import unit_cell
uc_fin = unit_cell((a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin))
crystal_init_orientation = pres_in.crystal_orientation
two_theta = observations_original.two_theta(wavelength=wavelength).data()
ph = partiality_handler()
partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(uc_fin, rotx_fin, roty_fin,
observations_original.indices(),
ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,
spot_pred_x_mm, spot_pred_y_mm,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
#calculate the new crystal orientation
O = sqr(uc_fin.orthogonalization_matrix()).transpose()
R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose()
from cctbx.crystal_orientation import crystal_orientation, basis_type
CO = crystal_orientation(O*R, basis_type.direct)
crystal_fin_orientation = CO.rotate_thru((1,0,0), rotx_fin
).rotate_thru((0,1,0), roty_fin)
#remove reflections with partiality below threshold
i_sel = partiality_fin > iparams.merge.partiality_min
partiality_fin_sel = partiality_fin.select(i_sel)
rs_fin_sel = rs_fin.select(i_sel)
rh_fin_sel = rh_fin.select(i_sel)
observations_non_polar_sel = observations_non_polar.select(i_sel)
observations_original_sel = observations_original.select(i_sel)
mapped_predictions = mapped_predictions.select(i_sel)
pres = postref_results()
pres.set_params(observations = observations_non_polar_sel,
observations_original = observations_original_sel,
refined_params=refined_params,
stats=stats,
partiality=partiality_fin_sel,
rs_set=rs_fin_sel,
rh_set=rh_fin_sel,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_init_orientation,
detector_distance_mm=detector_distance_mm,
identified_isoform=identified_isoform,
mapped_predictions=mapped_predictions,
xbeam=xbeam,
ybeam=ybeam)
r_change, r_xy_change, cc_change, cc_iso_change = (0,0,0,0)
try:
r_change = ((pres.R_final - pres.R_init)/pres.R_init)*100
r_xy_change = ((pres.R_xy_final - pres.R_xy_init)/pres.R_xy_init)*100
cc_change = ((pres.CC_final - pres.CC_init)/pres.CC_init)*100
cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init)/pres.CC_iso_init)*100
except Exception:
pass
txt_postref= ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}'.format(img_filename_only+' ('+index_basis_name+')', observations_original_sel.d_min(), len(observations_original_sel.data()), r_change, r_xy_change, cc_change, cc_iso_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin)
print txt_postref
txt_postref += '\n'
return pres, txt_postref
def postrefine_by_frame(self, frame_no, pickle_filename, iph, miller_array_ref):
#1. Prepare data
observations_pickle = pickle.load(open(pickle_filename,"rb"))
crystal_init_orientation = observations_pickle["current_orientation"][0]
wavelength = observations_pickle["wavelength"]
#grab img. name
imgname = pickle_filename
if iph.file_name_in_img != '':
fh = file_handler()
imgname = fh.get_imgname_from_pickle_filename(iph.file_name_in_img, pickle_filename)
observations_original, alpha_angle_obs = self.organize_input(observations_pickle, iph)
if observations_original is None:
print frame_no, '-fail obs is none'
return None
#2. Determine polarity - always do this even if flag_polar = False
#the function will take care of it.
polar_hkl, cc_iso_raw_asu, cc_iso_raw_rev = self.determine_polar(observations_original, iph, pickle_filename)
#3. Select data for post-refinement (only select indices that are common with the reference set
observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_ref.indices(),
miller_indices=observations_non_polar.indices())
I_ref_match = flex.double([miller_array_ref.data()[pair[0]] for pair in matches.pairs()])
miller_indices_ref_match = flex.miller_index((miller_array_ref.indices()[pair[0]] for pair in matches.pairs()))
I_obs_match = flex.double([observations_non_polar.data()[pair[1]] for pair in matches.pairs()])
sigI_obs_match = flex.double([observations_non_polar.sigmas()[pair[1]] for pair in matches.pairs()])
miller_indices_original_obs_match = flex.miller_index((observations_original.indices()[pair[1]] for pair in matches.pairs()))
alpha_angle_set = flex.double([alpha_angle_obs[pair[1]] for pair in matches.pairs()])
references_sel = miller_array_ref.customized_copy(data=I_ref_match, indices=miller_indices_ref_match)
observations_original_sel = observations_original.customized_copy(data=I_obs_match,
sigmas=sigI_obs_match,
indices=miller_indices_original_obs_match)
#4. Do least-squares refinement
lsqrh = leastsqr_handler()
refined_params, se_params, stats, partiality_sel, SE_I, var_I_p, var_k, var_p = lsqrh.optimize(I_ref_match, observations_original_sel,
wavelength, crystal_init_orientation, alpha_angle_set, iph)
if SE_I is None:
print 'frame', frame_no, ' - failed'
return None
else:
pres = postref_results()
observations_non_polar_sel = self.get_observations_non_polar(observations_original_sel, polar_hkl)
pres.set_params(observations = observations_non_polar_sel,
refined_params=refined_params,
se_params=se_params,
stats=stats,
partiality=partiality_sel,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
SE_I=SE_I,
var_I_p=var_I_p,
var_k=var_k,
var_p=var_p)
print 'frame %6.0f'%pres.frame_no, ' SE=%7.2f R-sq=%7.2f CC=%7.2f'%pres.stats, polar_hkl
return pres
def scale_0(self, filename):
"""
Perform 0th-order scaling.
"""
observations_hdlr = observations_handler(self.iparams)
observations_hdlr.init_params_from_file(filename)
"""
#get the miller_array_template
uc = self.iparams.target_unit_cell.parameters()
crystal_symmetry = crystal.symmetry(
unit_cell=(uc[0], uc[1], uc[2], uc[3], uc[4], uc[5]),
space_group_symbol=self.iparams.target_space_group
)
miller_array_complete = crystal_symmetry.build_miller_set(anomalous_flag=self.iparams.target_anomalous_flag,
d_min=self.iparams.d_min, d_max=self.iparams.d_max).array()
miller_array_complete = miller_array_complete.set_observation_type_xray_intensity()
miller_array_complete = miller_array_complete.customized_copy(data=flex.double([1e4]*len(miller_array_complete.indices())),
sigmas=flex.double([1e2]*len(miller_array_complete.indices())))
asu_contents = {}
asu_volume = miller_array_complete.unit_cell().volume()/float(miller_array_complete.space_group().order_z())
number_carbons = asu_volume/18.0
asu_contents.setdefault('C', number_carbons)
ftmpl = miller_array_complete.as_amplitude_array()
ftmpl.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(ftmpl, asu_contents, e_statistics=True)
G, B = (wp.wilson_intensity_scale_factor*1e3,wp.wilson_b)
stolsq = miller_array_complete.two_theta(observations_hdlr.wavelength).sin_theta_over_lambda_sq().data()
I_wp = miller_array_complete.data()/(G * flex.exp(-2 * B * stolsq))
sigI_wp = miller_array_complete.sigmas()/(G * flex.exp(-2 * B * stolsq))
miller_array_complete = miller_array_complete.customized_copy(data=I_wp, sigmas=sigI_wp)
miller_array_hires = miller_array_complete.resolution_filter(d_min=self.iparams.d_min, d_max=self.scale_0_d_max)
miller_array_lowres = miller_array_complete.resolution_filter(d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
obs_lowres = observations_hdlr.observations.resolution_filter(d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
G_lowres = obs_lowres.mean()/miller_array_lowres.mean()
observations_hdlr.set_params(G_lowres=G_lowres)
"""
from mod_util import utility_handler
util_hdlr = utility_handler()
flag_hklisoin_found, miller_array_iso = util_hdlr.get_miller_array_from_mtz(
self.iparams.hklisoin)
miller_array_complete = miller_array_iso.as_intensity_array()
miller_array_complete = miller_array_complete.customized_copy(
data=miller_array_complete.data(),
sigmas=miller_array_complete.sigmas())
if self.iparams.target_anomalous_flag:
miller_array_complete = miller_array_complete.generate_bijvoet_mates(
)
miller_array_hires = miller_array_complete.resolution_filter(
d_min=self.iparams.d_min, d_max=self.scale_0_d_max)
miller_array_lowres = miller_array_complete.resolution_filter(
d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
observations_hdlr_ref = observations_handler(self.iparams)
observations_hdlr_ref.init_params_from_merge(miller_array_hires)
G, B = (1, 0)
#caculate G and B-tensor for high resolution reflections
if True:
observations_full = observations_hdlr.calc_full_observations()
lsqr_hdlr = leastsqr_handler(self.iparams)
lsqr_hdlr.optimize_scalefactors(observations_hdlr_ref,
observations_hdlr,
use_binning=False)
#except Exception:
# return None, ' {0:40} ==> SCALING FAILED'.format(observations_hdlr.pickle_filename_only)
#calculate G for low-resolution
obs_lowres = observations_hdlr.observations.resolution_filter(
d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
obs_lowres = obs_lowres.select(
flex.bool(
np.absolute((obs_lowres.data() - np.mean(obs_lowres.data())) /
np.std(obs_lowres.data())) < self.sigma_I_cutoff))
G_lowres = obs_lowres.mean() / miller_array_lowres.mean()
observations_hdlr.set_params(G_lowres=G_lowres)
txt_out = ' {0:40} ==> SCALED RES:{1:5.2f} NREFL:{2:5d} SG:{12:8} Glow:{15:6.1f} G:{3:6.1f} B:{4:6.1f} {5:6.1f} {6:6.1f} {7:6.1f} {8:6.1f} {9:6.1f} Rfin:{11:5.2f} Gwp:{13:6.2f} Bwp{14:6.1f}'.format(
observations_hdlr.pickle_filename_only,
observations_hdlr.observations.d_min(),
len(observations_hdlr.observations.data()), observations_hdlr.G,
observations_hdlr.b11, observations_hdlr.b22,
observations_hdlr.b33, observations_hdlr.b12,
observations_hdlr.b13, observations_hdlr.b23,
observations_hdlr.r_init, observations_hdlr.r_final,
observations_hdlr.observations.space_group_info(), G, B, G_lowres)
return observations_hdlr, txt_out
def postrefine_by_frame(self, frame_no, pres_in, iparams, miller_array_ref,
avg_mode):
#Prepare data
if pres_in is None:
return None, 'Found empty pickle file'
observations_pickle = pickle.load(open(pres_in.pickle_filename, "rb"))
wavelength = observations_pickle["wavelength"]
crystal_init_orientation = observations_pickle["current_orientation"][
0]
pickle_filename = pres_in.pickle_filename
pickle_filepaths = pickle_filename.split('/')
img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1]
txt_exception = ' {0:40} ==> '.format(img_filename_only)
inputs, txt_organize_input = self.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pickle_filename)
if inputs is not None:
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
else:
txt_exception += txt_organize_input + '\n'
return None, txt_exception
#Select data for post-refinement (only select indices that are common with the reference set
observations_non_polar, index_basis_name = self.get_observations_non_polar(
observations_original, pickle_filename, iparams)
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_ref.indices(),
miller_indices=observations_non_polar.indices())
I_ref_match = flex.double(
[miller_array_ref.data()[pair[0]] for pair in matches.pairs()])
miller_indices_ref_match = flex.miller_index(
(miller_array_ref.indices()[pair[0]] for pair in matches.pairs()))
I_obs_match = flex.double([
observations_non_polar.data()[pair[1]] for pair in matches.pairs()
])
sigI_obs_match = flex.double([
observations_non_polar.sigmas()[pair[1]]
for pair in matches.pairs()
])
miller_indices_original_obs_match = flex.miller_index((observations_original.indices()[pair[1]] \
for pair in matches.pairs()))
miller_indices_non_polar_obs_match = flex.miller_index((observations_non_polar.indices()[pair[1]] \
for pair in matches.pairs()))
alpha_angle_set = flex.double(
[alpha_angle[pair[1]] for pair in matches.pairs()])
spot_pred_x_mm_set = flex.double(
[spot_pred_x_mm[pair[1]] for pair in matches.pairs()])
spot_pred_y_mm_set = flex.double(
[spot_pred_y_mm[pair[1]] for pair in matches.pairs()])
references_sel = miller_array_ref.customized_copy(
data=I_ref_match, indices=miller_indices_ref_match)
observations_original_sel = observations_original.customized_copy(
data=I_obs_match,
sigmas=sigI_obs_match,
indices=miller_indices_original_obs_match)
observations_non_polar_sel = observations_non_polar.customized_copy(
data=I_obs_match,
sigmas=sigI_obs_match,
indices=miller_indices_non_polar_obs_match)
#Do least-squares refinement
lsqrh = leastsqr_handler()
try:
refined_params, stats, n_refl_postrefined = lsqrh.optimize(
I_ref_match, observations_original_sel, wavelength,
crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set,
spot_pred_y_mm_set, iparams, pres_in,
observations_non_polar_sel, detector_distance_mm)
except Exception:
txt_exception += 'optimization failed.\n'
return None, txt_exception
#caculate partiality for output (with target_anomalous check)
G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \
a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params
inputs, txt_organize_input = self.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pickle_filename)
observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
observations_non_polar, index_basis_name = self.get_observations_non_polar(
observations_original, pickle_filename, iparams)
from cctbx.uctbx import unit_cell
uc_fin = unit_cell(
(a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin))
crystal_init_orientation = pres_in.crystal_orientation
two_theta = observations_original.two_theta(
wavelength=wavelength).data()
ph = partiality_handler()
partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(
uc_fin, rotx_fin, roty_fin, observations_original.indices(),
ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, two_theta,
alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm,
spot_pred_y_mm, detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
#calculate the new crystal orientation
O = sqr(uc_fin.orthogonalization_matrix()).transpose()
R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose()
from cctbx.crystal_orientation import crystal_orientation, basis_type
CO = crystal_orientation(O * R, basis_type.direct)
crystal_fin_orientation = CO.rotate_thru(
(1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin)
#remove reflections with partiality below threshold
i_sel = partiality_fin > iparams.merge.partiality_min
partiality_fin_sel = partiality_fin.select(i_sel)
rs_fin_sel = rs_fin.select(i_sel)
rh_fin_sel = rh_fin.select(i_sel)
observations_non_polar_sel = observations_non_polar.select(i_sel)
observations_original_sel = observations_original.select(i_sel)
mapped_predictions = mapped_predictions.select(i_sel)
pres = postref_results()
pres.set_params(observations=observations_non_polar_sel,
observations_original=observations_original_sel,
refined_params=refined_params,
stats=stats,
partiality=partiality_fin_sel,
rs_set=rs_fin_sel,
rh_set=rh_fin_sel,
frame_no=frame_no,
pickle_filename=pickle_filename,
wavelength=wavelength,
crystal_orientation=crystal_init_orientation,
detector_distance_mm=detector_distance_mm,
identified_isoform=identified_isoform,
mapped_predictions=mapped_predictions,
xbeam=xbeam,
ybeam=ybeam)
r_change, r_xy_change, cc_change, cc_iso_change = (0, 0, 0, 0)
try:
r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100
r_xy_change = (
(pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100
cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100
cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init) /
pres.CC_iso_init) * 100
except Exception:
pass
txt_postref = ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}'.format(
img_filename_only + ' (' + index_basis_name + ')',
observations_original_sel.d_min(),
len(observations_original_sel.data()), r_change, r_xy_change,
cc_change, cc_iso_change, pres.G, pres.B, a_fin, b_fin, c_fin,
alpha_fin, beta_fin, gamma_fin)
print txt_postref
txt_postref += '\n'
return pres, txt_postref
