def compute_functional_and_gradients(self):
lp_h = lbfgs_partiality_handler()
#calculate sum_sqr of the function
fvec = lp_h.func(self.x, self.args)
self.f = flex.sum(fvec * fvec)
#calculate gradient for each parameter
DELTA = 1.E-7
self.g = flex.double()
for x in xrange(self.n):
templist = list(self.x)
templist[x] += DELTA
dvalues = flex.double(templist)
dfvec = lp_h.func(dvalues, self.args)
df = flex.sum(dfvec * dfvec)
#calculate by finite_difference
self.g.append((df - self.f) / DELTA)
return self.f, self.g
def optimize(self, I_r_flex, observations_original, wavelength,
crystal_init_orientation, alpha_angle, spot_pred_x_mm,
spot_pred_y_mm, iparams, pres_in, observations_non_polar,
detector_distance_mm):
ph = partiality_handler()
lph = lbfgs_partiality_handler()
if iparams.postref.allparams.flag_on:
refine_steps = ['allparams']
else:
refine_steps = ['crystal_orientation']
if iparams.postref.reflecting_range.flag_on:
refine_steps.append('reflecting_range')
if iparams.postref.unit_cell.flag_on:
refine_steps.append('unit_cell')
#get miller array iso, if given.
miller_array_iso = None
#prepare data
pr_d_min = iparams.postref.allparams.d_min
pr_d_max = iparams.postref.allparams.d_max
pr_sigma_min = iparams.postref.allparams.sigma_min
pr_partiality_min = iparams.postref.allparams.partiality_min
pr_uc_tol = iparams.postref.allparams.uc_tolerance
cs = observations_original.crystal_symmetry().space_group(
).crystal_system()
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
#initialize values only in the first sub cycle and the first refine step.
spot_radius = ph.calc_spot_radius(
sqr(crystal_init_orientation.reciprocal_matrix()),
observations_original_sel.indices(), wavelength)
if pres_in is None:
ry, rz, r0, re, voigt_nu, rotx, roty = 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, 0.0, 0.0
#apply constrain on the unit cell using crystal system
uc_scale_inp = lph.prep_input(
observations_original.unit_cell().parameters(), cs)
uc_scale_constrained = lph.prep_output(uc_scale_inp, cs)
a, b, c, alpha, beta, gamma = uc_scale_constrained
const_params_scale = (rotx, roty, ry, rz, r0, re, voigt_nu, a, b,
c, alpha, beta, gamma)
xopt_scalefactors, stats = self.optimize_scalefactors(
I_r_flex, observations_original, wavelength,
crystal_init_orientation, alpha_angle, spot_pred_x_mm,
spot_pred_y_mm, iparams, pres_in, observations_non_polar,
detector_distance_mm, const_params_scale)
G, B = xopt_scalefactors
else:
G, B, ry, rz, r0, re, voigt_nu, rotx, roty = pres_in.G, pres_in.B, pres_in.ry, pres_in.rz, pres_in.r0, pres_in.re, pres_in.voigt_nu, 0.0, 0.0
a, b, c, alpha, beta, gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#filter by partiality
two_theta = observations_original_sel.two_theta(
wavelength=wavelength).data()
uc = unit_cell((a, b, c, alpha, beta, gamma))
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(
uc, rotx, roty, observations_original_sel.indices(), ry, rz, r0,
re, voigt_nu, two_theta, alpha_angle_sel, wavelength,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, iparams.partiality_model,
iparams.flag_beam_divergence)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'partiality', [pr_partiality_min], observations_original_sel,
alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel,
partiality_in=partiality_init)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()[:]
#calculate initial residual_xy error
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell', const_params_uc, B,
miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
init_residual_xy_err = flex.sum(uc_params_err**2)
#calculate initial residual_pr error
const_params_all = (G, B)
xinp_all = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp_all.extend(lph.prep_input((a, b, c, alpha, beta, gamma), cs))
args_all = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'allparams', const_params_all, B,
miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
init_residual_err = flex.sum(all_params_err**2)
#keep in list
t_pr_list = [init_residual_err]
t_xy_list = [init_residual_xy_err]
refined_params_hist = [(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,
b, c, alpha, beta, gamma)]
txt_out = ''
for i_sub_cycle in range(iparams.n_postref_sub_cycle):
for j_refine_step in range(len(refine_steps)):
refine_mode = refine_steps[j_refine_step]
#prepare data
init_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,
b, c, alpha, beta, gamma)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.prepare_data_microcycle(refine_mode, iparams,
observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm,
I_r_flex, init_params, crystal_init_orientation,
wavelength, detector_distance_mm)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()
if refine_mode == 'crystal_orientation':
xinp = flex.double([rotx, roty])
const_params = (G, B, ry, rz, r0, re, voigt_nu, a, b, c,
alpha, beta, gamma)
elif refine_mode == 'reflecting_range':
xinp = flex.double([ry, rz, r0, re, voigt_nu])
const_params = (G, B, rotx, roty, a, b, c, alpha, beta,
gamma)
elif refine_mode == 'unit_cell':
xinp = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
const_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
elif refine_mode == 'allparams':
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(
lph.prep_input((a, b, c, alpha, beta, gamma), cs))
const_params = (G, B)
args = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, refine_mode, const_params, B,
miller_array_iso, iparams)
lh = lbfgs_handler(current_x=xinp, args=args)
xopt = flex.double(list(lh.x))
if refine_mode == 'crystal_orientation' or \
refine_mode == 'reflecting_range' or refine_mode == 'allparams':
current_residual_err = lh.f
#calculate residual_xy_error (for refine_mode = SF, CO, RR, and all params)
xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
if refine_mode == 'crystal_orientation':
rotx, roty = xopt
elif refine_mode == 'reflecting_range':
ry, rz, r0, re, voigt_nu = xopt
elif refine_mode == 'allparams':
rotx, roty, ry, rz, r0, re, voigt_nu = xopt[:7]
xinp_uc = xopt[7:]
a, b, c, alpha, beta, gamma = lph.prep_output(
xinp_uc, cs)
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re,
voigt_nu)
xinp_uc = lph.prep_input((a, b, c, alpha, beta, gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength,
alpha_angle_sel, crystal_init_orientation,
spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell',
const_params_uc, B, miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
current_residual_xy_err = flex.sum(uc_params_err**2)
elif refine_mode == 'unit_cell':
current_residual_xy_err = lh.f
xopt_uc = lph.prep_output(xopt, cs)
a, b, c, alpha, beta, gamma = xopt_uc
#check the unit-cell with the reference intensity
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(
lph.prep_input((a, b, c, alpha, beta, gamma), cs))
const_params_all = (G, B)
args_all = (I_r_true, observations_original_sel,
wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel,
spot_pred_y_mm_sel, detector_distance_mm,
'allparams', const_params_all, B,
miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
current_residual_err = flex.sum(all_params_err**2)
flag_success = False
if refine_mode == 'allparams':
#if allparams refinement, only check the post-refine target function
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append(
(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b,
c, alpha, beta, gamma))
flag_success = True
else:
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
if current_residual_xy_err < (t_xy_list[len(t_xy_list)-1] + \
(t_xy_list[len(t_xy_list)-1]*iparams.postref.residual_threshold_xy/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append(
(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,
b, c, alpha, beta, gamma))
flag_success = True
if flag_success is False:
G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma = refined_params_hist[
len(refined_params_hist) - 1]
tmp_txt_out = refine_mode + ' %3.0f %6.4f %6.4f %6.4f %6.4f %10.8f %10.8f %10.8f %10.8f %10.8f %6.3f %6.3f %.4g %6.3f\n' % (
i_sub_cycle, G, B, rotx * 180 / math.pi, roty * 180 /
math.pi, ry, rz, r0, re, voigt_nu, a, c, t_pr_list[
len(t_pr_list) - 1], t_xy_list[len(t_pr_list) - 1])
txt_out += tmp_txt_out
#apply the refined parameters on the full (original) reflection set
two_theta = observations_original.two_theta(
wavelength=wavelength).data()
sin_theta_over_lambda_sq = observations_original.two_theta(
wavelength=wavelength).sin_theta_over_lambda_sq().data()
if pres_in is None:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
observations_original.unit_cell(),0.0, 0.0,observations_original.indices(),
0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, 1, 0,
partiality_init, rs_init)
else:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
pres_in.unit_cell,0.0, 0.0,observations_original.indices(),
pres_in.ry, pres_in.rz,pres_in.r0, pres_in.re, pres_in.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, pres_in.G,
pres_in.B, partiality_init, rs_init)
partiality_fin, delta_xy_fin, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(\
unit_cell((a,b,c,alpha,beta,gamma)),rotx, roty,observations_original.indices(),
ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength,crystal_init_orientation,
spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_fin = ph.calc_full_refl(observations_original.data(),
sin_theta_over_lambda_sq, G, B,
partiality_fin, rs_fin)
SE_of_the_estimate = standard_error_of_the_estimate(
I_r_flex, I_o_fin, 13)
R_sq = coefficient_of_determination(I_r_flex, I_o_fin) * 100
CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient()
CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient()
err_init = (I_r_flex - I_o_init) / observations_original.sigmas()
R_init = math.sqrt(flex.sum(err_init**2))
err_final = (I_r_flex - I_o_fin) / observations_original.sigmas()
R_final = math.sqrt(flex.sum(err_final**2))
R_xy_init = math.sqrt(flex.sum(delta_xy_init**2))
R_xy_final = math.sqrt(flex.sum(delta_xy_fin**2))
if R_init < R_final or re > (iparams.gamma_e * 3):
CC_final = CC_init
R_final = R_init
R_xy_final = R_xy_init
if pres_in is None:
G, B, r0, ry, rz, re, rotx, roty = (1.0, 0.0, spot_radius, 0.0,
0.0, iparams.gamma_e, 0.0,
0.0)
a, b, c, alpha, beta, gamma = observations_original.unit_cell(
).parameters()
else:
G, B, r0, ry, rz, re, rotx, roty = (pres_in.G, pres_in.B,
pres_in.r0, pres_in.ry,
pres_in.rz, pres_in.re,
0.0, 0.0)
a, b, c, alpha, beta, gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#calculate CCiso if hklisoin is given
CC_iso_init, CC_iso_final = (0, 0)
if iparams.hklisoin is not None:
if miller_array_iso is not None:
from cctbx import miller
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_iso.indices(),
miller_indices=observations_non_polar.indices())
I_iso_match = flex.double([
miller_array_iso.data()[pair[0]]
for pair in matches.pairs()
])
I_o_init_match = flex.double(
[I_o_init[pair[1]] for pair in matches.pairs()])
I_o_fin_match = flex.double(
[I_o_fin[pair[1]] for pair in matches.pairs()])
CC_iso_init = flex.linear_correlation(
I_iso_match, I_o_init_match).coefficient()
CC_iso_final = flex.linear_correlation(
I_iso_match, I_o_fin_match).coefficient()
xopt = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha,
beta, gamma)
return xopt, (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init,
R_final, R_xy_init, R_xy_final, CC_iso_init,
CC_iso_final), len(I_ref_sel)
def optimize(self, I_r_flex, observations_original,
wavelength, crystal_init_orientation,
alpha_angle, spot_pred_x_mm, spot_pred_y_mm, iparams,
pres_in, observations_non_polar, detector_distance_mm):
ph = partiality_handler()
lph = lbfgs_partiality_handler()
if iparams.postref.allparams.flag_on:
refine_steps = ['allparams']
else:
refine_steps = ['crystal_orientation']
if iparams.postref.reflecting_range.flag_on:
refine_steps.append('reflecting_range')
if iparams.postref.unit_cell.flag_on:
refine_steps.append('unit_cell')
#get miller array iso, if given.
miller_array_iso = None
#prepare data
pr_d_min = iparams.postref.allparams.d_min
pr_d_max = iparams.postref.allparams.d_max
pr_sigma_min = iparams.postref.allparams.sigma_min
pr_partiality_min = iparams.postref.allparams.partiality_min
pr_uc_tol = iparams.postref.allparams.uc_tolerance
cs = observations_original.crystal_symmetry().space_group().crystal_system()
#filter by resolution
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'resolution', [pr_d_min, pr_d_max], observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm, I_r_flex)
#filter by sigma
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'sigma', [pr_sigma_min], observations_original_sel, alpha_angle_sel,
spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel)
#initialize values only in the first sub cycle and the first refine step.
spot_radius = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()),
observations_original_sel.indices(), wavelength)
if pres_in is None:
ry, rz, r0, re, voigt_nu, rotx, roty = 0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu, 0.0, 0.0
#apply constrain on the unit cell using crystal system
uc_scale_inp = lph.prep_input(observations_original.unit_cell().parameters(), cs)
uc_scale_constrained = lph.prep_output(uc_scale_inp, cs)
a,b,c,alpha,beta,gamma = uc_scale_constrained
const_params_scale = (rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)
xopt_scalefactors, stats = self.optimize_scalefactors(I_r_flex,
observations_original,
wavelength, crystal_init_orientation,
alpha_angle,
spot_pred_x_mm,
spot_pred_y_mm,
iparams,
pres_in,
observations_non_polar,
detector_distance_mm,
const_params_scale)
G, B = xopt_scalefactors
else:
G, B, ry, rz, r0, re, voigt_nu, rotx, roty = pres_in.G, pres_in.B, pres_in.ry, pres_in.rz, pres_in.r0, pres_in.re, pres_in.voigt_nu, 0.0 , 0.0
a,b,c,alpha,beta,gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#filter by partiality
two_theta = observations_original_sel.two_theta(wavelength=wavelength).data()
uc = unit_cell((a,b,c,alpha,beta,gamma))
partiality_init, delta_xy_init, rs_init, dummy = ph.calc_partiality_anisotropy_set(uc, rotx, roty,
observations_original_sel.indices(),
ry, rz, r0, re, voigt_nu, two_theta,
alpha_angle_sel, wavelength,
crystal_init_orientation,
spot_pred_x_mm_sel,
spot_pred_y_mm_sel,
detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.get_filtered_data(\
'partiality', [pr_partiality_min], observations_original_sel,
alpha_angle_sel, spot_pred_x_mm_sel, spot_pred_y_mm_sel, I_ref_sel,
partiality_in=partiality_init)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()[:]
#calculate initial residual_xy error
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
init_residual_xy_err = flex.sum(uc_params_err**2)
#calculate initial residual_pr error
const_params_all= (G,B)
xinp_all = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp_all.extend(lph.prep_input((a,b,c,alpha,beta,gamma), cs))
args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
init_residual_err = flex.sum(all_params_err**2)
#keep in list
t_pr_list = [init_residual_err]
t_xy_list = [init_residual_xy_err]
refined_params_hist = [(G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)]
txt_out = ''
for i_sub_cycle in range(iparams.n_postref_sub_cycle):
for j_refine_step in range(len(refine_steps)):
refine_mode = refine_steps[j_refine_step]
#prepare data
init_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)
observations_original_sel, alpha_angle_sel, spot_pred_x_mm_sel, \
spot_pred_y_mm_sel, I_ref_sel = self.prepare_data_microcycle(refine_mode, iparams,
observations_original, alpha_angle,
spot_pred_x_mm, spot_pred_y_mm,
I_r_flex, init_params, crystal_init_orientation,
wavelength, detector_distance_mm)
I_r_true = I_ref_sel[:]
I_o_true = observations_original_sel.data()
if refine_mode == 'crystal_orientation':
xinp = flex.double([rotx, roty])
const_params = (G, B, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma)
elif refine_mode == 'reflecting_range':
xinp = flex.double([ry, rz, r0, re, voigt_nu])
const_params = (G, B, rotx, roty, a, b, c, alpha, beta, gamma)
elif refine_mode == 'unit_cell':
xinp = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
const_params = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
elif refine_mode == 'allparams':
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(lph.prep_input((a,b,c,alpha,beta,gamma), cs))
const_params = (G,B)
args=(I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, refine_mode, const_params, B, miller_array_iso, iparams)
lh = lbfgs_handler(current_x=xinp, args=args)
xopt = flex.double(list(lh.x))
if refine_mode == 'crystal_orientation' or \
refine_mode == 'reflecting_range' or refine_mode == 'allparams':
current_residual_err = lh.f
#calculate residual_xy_error (for refine_mode = SF, CO, RR, and all params)
xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
if refine_mode == 'crystal_orientation':
rotx, roty = xopt
elif refine_mode == 'reflecting_range':
ry, rz, r0, re, voigt_nu = xopt
elif refine_mode == 'allparams':
rotx, roty, ry, rz, r0, re, voigt_nu = xopt[:7]
xinp_uc = xopt[7:]
a, b, c, alpha, beta, gamma = lph.prep_output(xinp_uc, cs)
const_params_uc = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu)
xinp_uc = lph.prep_input((a,b,c,alpha,beta,gamma), cs)
args_uc = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'unit_cell', const_params_uc, B, miller_array_iso, iparams)
uc_params_err = lph.func(xinp_uc, args_uc)
current_residual_xy_err = flex.sum(uc_params_err**2)
elif refine_mode == 'unit_cell':
current_residual_xy_err = lh.f
xopt_uc = lph.prep_output(xopt, cs)
a, b, c, alpha, beta, gamma = xopt_uc
#check the unit-cell with the reference intensity
xinp = flex.double([rotx, roty, ry, rz, r0, re, voigt_nu])
xinp.extend(lph.prep_input((a, b, c, alpha, beta, gamma), cs))
const_params_all = (G,B)
args_all = (I_r_true, observations_original_sel, wavelength, alpha_angle_sel,
crystal_init_orientation, spot_pred_x_mm_sel, spot_pred_y_mm_sel,
detector_distance_mm, 'allparams', const_params_all, B, miller_array_iso, iparams)
all_params_err = lph.func(xinp_all, args_all)
current_residual_err = flex.sum(all_params_err**2)
flag_success = False
if refine_mode == 'allparams':
#if allparams refinement, only check the post-refine target function
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append((G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma))
flag_success = True
else:
if current_residual_err < (t_pr_list[len(t_pr_list)-1] + \
(t_pr_list[len(t_pr_list)-1]*iparams.postref.residual_threshold/100)):
if current_residual_xy_err < (t_xy_list[len(t_xy_list)-1] + \
(t_xy_list[len(t_xy_list)-1]*iparams.postref.residual_threshold_xy/100)):
t_pr_list.append(current_residual_err)
t_xy_list.append(current_residual_xy_err)
refined_params_hist.append((G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a, b, c, alpha, beta, gamma))
flag_success = True
if flag_success is False:
G,B,rotx,roty,ry,rz,r0,re,voigt_nu,a,b,c,alpha,beta,gamma = refined_params_hist[len(refined_params_hist)-1]
tmp_txt_out = refine_mode + ' %3.0f %6.4f %6.4f %6.4f %6.4f %10.8f %10.8f %10.8f %10.8f %10.8f %6.3f %6.3f %.4g %6.3f\n'%(i_sub_cycle,G,B,rotx*180/math.pi,roty*180/math.pi,ry,rz,r0,re,voigt_nu,a,c,t_pr_list[len(t_pr_list)-1],t_xy_list[len(t_pr_list)-1])
txt_out += tmp_txt_out
#apply the refined parameters on the full (original) reflection set
two_theta = observations_original.two_theta(wavelength=wavelength).data()
sin_theta_over_lambda_sq = observations_original.two_theta(wavelength=wavelength).sin_theta_over_lambda_sq().data()
if pres_in is None:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
observations_original.unit_cell(),0.0, 0.0,observations_original.indices(),
0, 0, spot_radius, iparams.gamma_e, iparams.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
1, 0, partiality_init, rs_init)
else:
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
pres_in.unit_cell,0.0, 0.0,observations_original.indices(),
pres_in.ry, pres_in.rz,pres_in.r0, pres_in.re, pres_in.voigt_nu,
two_theta, alpha_angle, wavelength,
crystal_init_orientation,spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_init = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
pres_in.G, pres_in.B, partiality_init, rs_init)
partiality_fin, delta_xy_fin, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(\
unit_cell((a,b,c,alpha,beta,gamma)),rotx, roty,observations_original.indices(),
ry, rz, r0, re, voigt_nu, two_theta, alpha_angle, wavelength,crystal_init_orientation,
spot_pred_x_mm, spot_pred_y_mm,detector_distance_mm,
iparams.partiality_model,iparams.flag_beam_divergence)
I_o_fin = ph.calc_full_refl(observations_original.data(), sin_theta_over_lambda_sq,
G, B, partiality_fin, rs_fin)
SE_of_the_estimate = standard_error_of_the_estimate(I_r_flex,I_o_fin, 13)
R_sq = coefficient_of_determination(I_r_flex,I_o_fin)*100
CC_init = flex.linear_correlation(I_r_flex, I_o_init).coefficient()
CC_final = flex.linear_correlation(I_r_flex, I_o_fin).coefficient()
err_init = (I_r_flex - I_o_init)/observations_original.sigmas()
R_init = math.sqrt(flex.sum(err_init**2))
err_final = (I_r_flex - I_o_fin)/observations_original.sigmas()
R_final = math.sqrt(flex.sum(err_final**2))
R_xy_init = math.sqrt(flex.sum(delta_xy_init**2))
R_xy_final = math.sqrt(flex.sum(delta_xy_fin**2))
if R_init < R_final or re > (iparams.gamma_e * 10):
CC_final = CC_init
R_final = R_init
R_xy_final = R_xy_init
if pres_in is None:
G,B,r0,ry,rz,re,rotx,roty = (1.0,0.0,spot_radius,0.0,0.0,iparams.gamma_e,0.0,0.0)
a,b,c,alpha,beta,gamma = observations_original.unit_cell().parameters()
else:
G,B,r0,ry,rz,re,rotx,roty = (pres_in.G,pres_in.B,pres_in.r0,pres_in.ry,pres_in.rz,pres_in.re,0.0,0.0)
a,b,c,alpha,beta,gamma = pres_in.unit_cell.parameters()
crystal_init_orientation = pres_in.crystal_orientation
#calculate CCiso if hklisoin is given
CC_iso_init,CC_iso_final = (0,0)
if iparams.hklisoin is not None:
if miller_array_iso is not None:
from cctbx import miller
matches = miller.match_multi_indices(
miller_indices_unique=miller_array_iso.indices(),
miller_indices=observations_non_polar.indices())
I_iso_match = flex.double([miller_array_iso.data()[pair[0]] for pair in matches.pairs()])
I_o_init_match = flex.double([I_o_init[pair[1]] for pair in matches.pairs()])
I_o_fin_match = flex.double([I_o_fin[pair[1]] for pair in matches.pairs()])
CC_iso_init = flex.linear_correlation(I_iso_match, I_o_init_match).coefficient()
CC_iso_final = flex.linear_correlation(I_iso_match, I_o_fin_match).coefficient()
xopt = (G, B, rotx, roty, ry, rz, r0, re, voigt_nu, a,b,c,alpha,beta,gamma)
return xopt, (SE_of_the_estimate, R_sq, CC_init, CC_final, R_init, R_final, R_xy_init, R_xy_final, CC_iso_init, CC_iso_final), len(I_ref_sel)
def organize_input(self,
observations_pickle,
iparams,
avg_mode,
pickle_filename=None):
"""Given the pickle file, extract and prepare observations object and
the alpha angle (meridional to equatorial).
"""
#get general parameters
if iparams.isoform_name is not None:
if "identified_isoform" not in observations_pickle:
return None, "No identified isoform"
if observations_pickle[
"identified_isoform"] != iparams.isoform_name:
return None, "Identified isoform(%s) is not the requested isoform (%s)" % (
observations_pickle["identified_isoform"],
iparams.isoform_name)
if iparams.flag_weak_anomalous:
if avg_mode == 'final':
target_anomalous_flag = iparams.target_anomalous_flag
else:
target_anomalous_flag = False
else:
target_anomalous_flag = iparams.target_anomalous_flag
img_filename_only = ''
if pickle_filename:
img_filename_only = os.path.basename(pickle_filename)
txt_exception = ' {0:40} ==> '.format(img_filename_only)
#for dials integration pickles - also look for experimentxxx.json
if "miller_index" in observations_pickle:
from dxtbx.model.experiment_list import ExperimentListFactory
exp_json_file = os.path.join(
os.path.dirname(pickle_filename),
img_filename_only.split('_')[0] + '_refined_experiments.json')
if os.path.isfile(exp_json_file):
experiments = ExperimentListFactory.from_json_file(
exp_json_file)
dials_crystal = experiments[0].crystal
detector = experiments[0].detector
beam = experiments[0].beam
crystal_symmetry = crystal.symmetry(
unit_cell=dials_crystal.get_unit_cell().parameters(),
space_group_symbol=iparams.target_space_group)
miller_set_all = miller.set(
crystal_symmetry=crystal_symmetry,
indices=observations_pickle['miller_index'],
anomalous_flag=target_anomalous_flag)
observations = miller_set_all.array(
data=observations_pickle['intensity.sum.value'],
sigmas=flex.sqrt(
observations_pickle['intensity.sum.variance'])
).set_observation_type_xray_intensity()
detector_distance_mm = detector[0].get_distance()
alpha_angle_obs = flex.double([0] * len(observations.data()))
wavelength = beam.get_wavelength()
spot_pred_x_mm = observations_pickle['s1'] #a disguise of s1
spot_pred_y_mm = flex.double([0] * len(observations.data()))
#calculate the crystal orientation
O = sqr(dials_crystal.get_unit_cell().orthogonalization_matrix(
)).transpose()
R = sqr(dials_crystal.get_U()).transpose()
from cctbx.crystal_orientation import crystal_orientation, basis_type
crystal_init_orientation = crystal_orientation(
O * R, basis_type.direct)
else:
txt_exception += exp_json_file + ' not found'
print txt_exception
return None, txt_exception
else:
#for cctbx.xfel proceed as usual
observations = observations_pickle["observations"][0]
detector_distance_mm = observations_pickle['distance']
mm_predictions = iparams.pixel_size_mm * (
observations_pickle['mapped_predictions'][0])
xbeam = observations_pickle["xbeam"]
ybeam = observations_pickle["ybeam"]
alpha_angle_obs = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \
for pred in mm_predictions])
spot_pred_x_mm = flex.double(
[pred[0] - xbeam for pred in mm_predictions])
spot_pred_y_mm = flex.double(
[pred[1] - ybeam for pred in mm_predictions])
#Polarization correction
wavelength = observations_pickle["wavelength"]
crystal_init_orientation = observations_pickle[
"current_orientation"][0]
#continue reading...
if iparams.flag_LP_correction and "observations" in observations_pickle:
fx = 1 - iparams.polarization_horizontal_fraction
fy = 1 - fx
if fx > 1.0 or fx < 0:
print 'Horizontal polarization fraction is not correct. The value must be >= 0 and <= 1'
print 'No polarization correction. Continue with post-refinement'
else:
phi_angle_obs = flex.double([math.atan2(pred[1]-ybeam, pred[0]-xbeam) \
for pred in mm_predictions])
bragg_angle_obs = observations.two_theta(wavelength).data()
P = ((fx*((flex.sin(phi_angle_obs)**2)+((flex.cos(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2)))+\
(fy*((flex.cos(phi_angle_obs)**2)+((flex.sin(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2))))
I_prime = observations.data() / P
sigI_prime = observations.sigmas() / P
observations = observations.customized_copy(
data=flex.double(I_prime), sigmas=flex.double(sigI_prime))
#set observations with target space group - !!! required for correct
#merging due to map_to_asu command.
if iparams.target_crystal_system is not None:
target_crystal_system = iparams.target_crystal_system
else:
target_crystal_system = observations.crystal_symmetry(
).space_group().crystal_system()
lph = lbfgs_partiality_handler()
if iparams.flag_override_unit_cell:
uc_constrained_inp = lph.prep_input(
iparams.target_unit_cell.parameters(), target_crystal_system)
else:
uc_constrained_inp = lph.prep_input(
observations.unit_cell().parameters(), target_crystal_system)
uc_constrained = list(
lph.prep_output(uc_constrained_inp, target_crystal_system))
try:
#apply constrain using the crystal system
miller_set = symmetry(unit_cell=uc_constrained,
space_group_symbol=iparams.target_space_group
).build_miller_set(
anomalous_flag=target_anomalous_flag,
d_min=iparams.merge.d_min)
observations = observations.customized_copy(
anomalous_flag=target_anomalous_flag,
crystal_symmetry=miller_set.crystal_symmetry())
except Exception:
a, b, c, alpha, beta, gamma = uc_constrained
txt_exception += 'Mismatch spacegroup (%6.2f,%6.2f,%6.2f,%6.2f,%6.2f,%6.2f)' % (
a, b, c, alpha, beta, gamma)
print txt_exception
return None, txt_exception
#reset systematic absence
sys_absent_negate_flags = flex.bool([
sys_absent_flag[1] == False
for sys_absent_flag in observations.sys_absent_flags()
])
observations = observations.select(sys_absent_negate_flags)
alpha_angle_obs = alpha_angle_obs.select(sys_absent_negate_flags)
spot_pred_x_mm = spot_pred_x_mm.select(sys_absent_negate_flags)
spot_pred_y_mm = spot_pred_y_mm.select(sys_absent_negate_flags)
#remove observations from rejection list
if os.path.isfile(iparams.run_no + '/rejections.txt'):
txt_out = pickle_filename + ' \nN_before_rejection: ' + str(
len(observations.data())) + '\n'
file_reject = open(iparams.run_no + '/rejections.txt', 'r')
data_reject = file_reject.read().split("\n")
miller_indices_ori_rejected = flex.miller_index()
for row_reject in data_reject:
col_reject = row_reject.split()
if len(col_reject) > 0:
if col_reject[0].strip() == pickle_filename:
miller_indices_ori_rejected.append(
(int(col_reject[1].strip()),
int(col_reject[2].strip()),
int(col_reject[3].strip())))
if len(miller_indices_ori_rejected) > 0:
i_sel_flag = flex.bool([True] * len(observations.data()))
for miller_index_ori_rejected in miller_indices_ori_rejected:
i_index_ori = 0
for miller_index_ori in observations.indices():
if miller_index_ori_rejected == miller_index_ori:
i_sel_flag[i_index_ori] = False
txt_out += ' -Discard:' + str(miller_index_ori[0]) + \
','+str(miller_index_ori[1])+','+str(miller_index_ori[2]) + '\n'
i_index_ori += 1
observations = observations.customized_copy(
indices=observations.indices().select(i_sel_flag),
data=observations.data().select(i_sel_flag),
sigmas=observations.sigmas().select(i_sel_flag))
alpha_angle_obs = alpha_angle_obs.select(i_sel_flag)
spot_pred_x_mm = spot_pred_x_mm.select(i_sel_flag)
spot_pred_y_mm = spot_pred_y_mm.select(i_sel_flag)
txt_out += 'N_after_rejection: ' + str(len(
observations.data())) + '\n'
#filter resolution
i_sel_res = observations.resolution_filter_selection(
d_max=iparams.merge.d_max, d_min=iparams.merge.d_min)
observations = observations.select(i_sel_res)
alpha_angle_obs = alpha_angle_obs.select(i_sel_res)
spot_pred_x_mm = spot_pred_x_mm.select(i_sel_res)
spot_pred_y_mm = spot_pred_y_mm.select(i_sel_res)
#Filter weak
i_sel = (observations.data() /
observations.sigmas()) > iparams.merge.sigma_min
observations = observations.select(i_sel)
alpha_angle_obs = alpha_angle_obs.select(i_sel)
spot_pred_x_mm = spot_pred_x_mm.select(i_sel)
spot_pred_y_mm = spot_pred_y_mm.select(i_sel)
#filter icering (if on)
if iparams.icering.flag_on:
miller_indices = flex.miller_index()
I_set = flex.double()
sigI_set = flex.double()
alpha_angle_obs_set = flex.double()
spot_pred_x_mm_set = flex.double()
spot_pred_y_mm_set = flex.double()
for miller_index, d, I, sigI, alpha, spot_x, spot_y in zip(
observations.indices(),
observations.d_spacings().data(), observations.data(),
observations.sigmas(), alpha_angle_obs, spot_pred_x_mm,
spot_pred_y_mm):
if d > iparams.icering.d_upper or d < iparams.icering.d_lower:
miller_indices.append(miller_index)
I_set.append(I)
sigI_set.append(sigI)
alpha_angle_obs_set.append(alpha)
spot_pred_x_mm_set.append(spot_x)
spot_pred_y_mm_set.append(spot_y)
observations = observations.customized_copy(indices=miller_indices,
data=I_set,
sigmas=sigI_set)
alpha_angle_obs = alpha_angle_obs_set[:]
spot_pred_x_mm = spot_pred_x_mm_set[:]
spot_pred_y_mm = spot_pred_y_mm_set[:]
#replacing sigI (if set)
if iparams.flag_replace_sigI:
observations = observations.customized_copy(
sigmas=flex.sqrt(observations.data()))
inputs = observations, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation
return inputs, 'OK'
def organize_input(self, observations_pickle, iparams, avg_mode, pickle_filename=None):
"""Given the pickle file, extract and prepare observations object and
the alpha angle (meridional to equatorial).
"""
identified_isoform = None
if iparams.isoform_name is not None:
identified_isoform = iparams.isoform_name
if "identified_isoform" not in observations_pickle:
return None, "No identified isoform"
else:
identified_isoform = observations_pickle["identified_isoform"]
if observations_pickle["identified_isoform"] != iparams.isoform_name:
return None, "Identified isoform(%s) is not the requested isoform (%s)"%(observations_pickle["identified_isoform"], iparams.isoform_name)
if iparams.flag_weak_anomalous:
if avg_mode == 'final':
target_anomalous_flag = iparams.target_anomalous_flag
else:
target_anomalous_flag = False
else:
target_anomalous_flag = iparams.target_anomalous_flag
img_filename_only = ''
if pickle_filename is not None:
pickle_filepaths = pickle_filename.split('/')
img_filename_only = pickle_filepaths[len(pickle_filepaths)-1]
txt_exception = ' {0:40} ==> '.format(img_filename_only)
observations = observations_pickle["observations"][0]
detector_distance_mm = observations_pickle['distance']
mapped_predictions = observations_pickle['mapped_predictions'][0]
#set observations with target space group - !!! required for correct
#merging due to map_to_asu command.
if iparams.target_crystal_system is not None:
target_crystal_system = iparams.target_crystal_system
else:
target_crystal_system = observations.crystal_symmetry().space_group().crystal_system()
lph = lbfgs_partiality_handler()
if iparams.flag_override_unit_cell:
uc_constrained_inp = lph.prep_input(iparams.target_unit_cell.parameters(), target_crystal_system)
else:
uc_constrained_inp = lph.prep_input(observations.unit_cell().parameters(), target_crystal_system)
uc_constrained = list(lph.prep_output(uc_constrained_inp, target_crystal_system))
try:
#apply constrain using the crystal system
miller_set = symmetry(
unit_cell=uc_constrained,
space_group_symbol=iparams.target_space_group
).build_miller_set(
anomalous_flag=target_anomalous_flag,
d_min=iparams.merge.d_min)
observations = observations.customized_copy(anomalous_flag=target_anomalous_flag,
crystal_symmetry=miller_set.crystal_symmetry())
except Exception:
a,b,c,alpha,beta,gamma = uc_constrained
txt_exception += 'Mismatch spacegroup (%6.2f,%6.2f,%6.2f,%6.2f,%6.2f,%6.2f)'%(a,b,c,alpha,beta,gamma)
return None, txt_exception
#reset systematic absence
sys_absent_negate_flags = flex.bool([sys_absent_flag[1]==False for sys_absent_flag in observations.sys_absent_flags()])
observations = observations.select(sys_absent_negate_flags)
mapped_predictions = mapped_predictions.select(sys_absent_negate_flags)
import os.path
#remove observations from rejection list
if os.path.isfile(iparams.run_no+'/rejections.txt'):
txt_out = pickle_filename + ' \nN_before_rejection: ' + str(len(observations.data())) + '\n'
file_reject = open(iparams.run_no+'/rejections.txt', 'r')
data_reject=file_reject.read().split("\n")
miller_indices_ori_rejected = flex.miller_index()
for row_reject in data_reject:
col_reject = row_reject.split()
if len(col_reject) > 0:
if col_reject[0].strip() == pickle_filename:
miller_indices_ori_rejected.append((int(col_reject[1].strip()), int(col_reject[2].strip()), int(col_reject[3].strip())))
if len(miller_indices_ori_rejected) > 0:
i_sel_flag = flex.bool([True]*len(observations.data()))
for miller_index_ori_rejected in miller_indices_ori_rejected:
i_index_ori = 0
for miller_index_ori in observations.indices():
if miller_index_ori_rejected == miller_index_ori:
i_sel_flag[i_index_ori] = False
txt_out += ' -Discard:' + str(miller_index_ori[0]) + \
','+str(miller_index_ori[1])+','+str(miller_index_ori[2]) + '\n'
i_index_ori += 1
observations = observations.customized_copy(indices=observations.indices().select(i_sel_flag),
data=observations.data().select(i_sel_flag),
sigmas=observations.sigmas().select(i_sel_flag))
mapped_predictions = mapped_predictions.select(i_sel_flag)
txt_out += 'N_after_rejection: ' + str(len(observations.data())) + '\n'
#filter resolution
i_sel_res = observations.resolution_filter_selection(d_max=iparams.merge.d_max, d_min=iparams.merge.d_min)
observations = observations.select(i_sel_res)
mapped_predictions = mapped_predictions.select(i_sel_res)
#Filter weak
i_sel = (observations.data()/observations.sigmas()) > iparams.merge.sigma_min
observations = observations.select(i_sel)
mapped_predictions = mapped_predictions.select(i_sel)
#filter icering (if on)
#replacing sigI (if set)
if iparams.flag_replace_sigI:
observations = observations.customized_copy(sigmas=flex.sqrt(observations.data()))
#setup spot predicton
mm_predictions = iparams.pixel_size_mm*mapped_predictions
xbeam = observations_pickle["xbeam"]
ybeam = observations_pickle["ybeam"]
alpha_angle_obs = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \
for pred in mm_predictions])
spot_pred_x_mm = flex.double([pred[0]-xbeam for pred in mm_predictions])
spot_pred_y_mm = flex.double([pred[1]-ybeam for pred in mm_predictions])
#Polarization correction
wavelength = observations_pickle["wavelength"]
if iparams.flag_LP_correction:
fx = 1 - iparams.polarization_horizontal_fraction
fy = 1 - fx
if fx > 1.0 or fx < 0:
print 'Horizontal polarization fraction is not correct. The value must be >= 0 and <= 1'
print 'No polarization correction. Continue with post-refinement'
else:
phi_angle_obs = flex.double([math.atan2(pred[1]-ybeam, pred[0]-xbeam) \
for pred in mm_predictions])
bragg_angle_obs = observations.two_theta(wavelength).data()
P = ((fx*((flex.sin(phi_angle_obs)**2)+((flex.cos(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2)))+\
(fy*((flex.cos(phi_angle_obs)**2)+((flex.sin(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2))))
I_prime = observations.data()/P
sigI_prime =observations.sigmas()/P
observations = observations.customized_copy(data=flex.double(I_prime),
sigmas=flex.double(sigI_prime))
inputs = observations, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, \
mapped_predictions, xbeam, ybeam
return inputs, 'OK'
def organize_input(self,
observations_pickle,
iparams,
avg_mode,
pickle_filename=None):
"""Given the pickle file, extract and prepare observations object and
the alpha angle (meridional to equatorial).
"""
identified_isoform = None
if iparams.isoform_name is not None:
identified_isoform = iparams.isoform_name
if "identified_isoform" not in observations_pickle:
return None, "No identified isoform"
else:
identified_isoform = observations_pickle["identified_isoform"]
if observations_pickle[
"identified_isoform"] != iparams.isoform_name:
return None, "Identified isoform(%s) is not the requested isoform (%s)" % (
observations_pickle["identified_isoform"],
iparams.isoform_name)
if iparams.flag_weak_anomalous:
if avg_mode == 'final':
target_anomalous_flag = iparams.target_anomalous_flag
else:
target_anomalous_flag = False
else:
target_anomalous_flag = iparams.target_anomalous_flag
img_filename_only = ''
if pickle_filename is not None:
pickle_filepaths = pickle_filename.split('/')
img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1]
txt_exception = ' {0:40} ==> '.format(img_filename_only)
observations = observations_pickle["observations"][0]
detector_distance_mm = observations_pickle['distance']
mapped_predictions = observations_pickle['mapped_predictions'][0]
#set observations with target space group - !!! required for correct
#merging due to map_to_asu command.
if iparams.target_crystal_system is not None:
target_crystal_system = iparams.target_crystal_system
else:
target_crystal_system = observations.crystal_symmetry(
).space_group().crystal_system()
lph = lbfgs_partiality_handler()
if iparams.flag_override_unit_cell:
uc_constrained_inp = lph.prep_input(
iparams.target_unit_cell.parameters(), target_crystal_system)
else:
uc_constrained_inp = lph.prep_input(
observations.unit_cell().parameters(), target_crystal_system)
uc_constrained = list(
lph.prep_output(uc_constrained_inp, target_crystal_system))
try:
#apply constrain using the crystal system
miller_set = symmetry(unit_cell=uc_constrained,
space_group_symbol=iparams.target_space_group
).build_miller_set(
anomalous_flag=target_anomalous_flag,
d_min=iparams.merge.d_min)
observations = observations.customized_copy(
anomalous_flag=target_anomalous_flag,
crystal_symmetry=miller_set.crystal_symmetry())
except Exception:
a, b, c, alpha, beta, gamma = uc_constrained
txt_exception += 'Mismatch spacegroup (%6.2f,%6.2f,%6.2f,%6.2f,%6.2f,%6.2f)' % (
a, b, c, alpha, beta, gamma)
return None, txt_exception
#reset systematic absence
sys_absent_negate_flags = flex.bool([
sys_absent_flag[1] == False
for sys_absent_flag in observations.sys_absent_flags()
])
observations = observations.select(sys_absent_negate_flags)
mapped_predictions = mapped_predictions.select(sys_absent_negate_flags)
import os.path
#remove observations from rejection list
if os.path.isfile(iparams.run_no + '/rejections.txt'):
txt_out = pickle_filename + ' \nN_before_rejection: ' + str(
len(observations.data())) + '\n'
file_reject = open(iparams.run_no + '/rejections.txt', 'r')
data_reject = file_reject.read().split("\n")
miller_indices_ori_rejected = flex.miller_index()
for row_reject in data_reject:
col_reject = row_reject.split()
if len(col_reject) > 0:
if col_reject[0].strip() == pickle_filename:
miller_indices_ori_rejected.append(
(int(col_reject[1].strip()),
int(col_reject[2].strip()),
int(col_reject[3].strip())))
if len(miller_indices_ori_rejected) > 0:
i_sel_flag = flex.bool([True] * len(observations.data()))
for miller_index_ori_rejected in miller_indices_ori_rejected:
i_index_ori = 0
for miller_index_ori in observations.indices():
if miller_index_ori_rejected == miller_index_ori:
i_sel_flag[i_index_ori] = False
txt_out += ' -Discard:' + str(miller_index_ori[0]) + \
','+str(miller_index_ori[1])+','+str(miller_index_ori[2]) + '\n'
i_index_ori += 1
observations = observations.customized_copy(
indices=observations.indices().select(i_sel_flag),
data=observations.data().select(i_sel_flag),
sigmas=observations.sigmas().select(i_sel_flag))
mapped_predictions = mapped_predictions.select(i_sel_flag)
txt_out += 'N_after_rejection: ' + str(len(
observations.data())) + '\n'
#filter resolution
i_sel_res = observations.resolution_filter_selection(
d_max=iparams.merge.d_max, d_min=iparams.merge.d_min)
observations = observations.select(i_sel_res)
mapped_predictions = mapped_predictions.select(i_sel_res)
#Filter weak
i_sel = (observations.data() /
observations.sigmas()) > iparams.merge.sigma_min
observations = observations.select(i_sel)
mapped_predictions = mapped_predictions.select(i_sel)
#filter icering (if on)
#replacing sigI (if set)
if iparams.flag_replace_sigI:
observations = observations.customized_copy(
sigmas=flex.sqrt(observations.data()))
#setup spot predicton
mm_predictions = iparams.pixel_size_mm * mapped_predictions
xbeam = observations_pickle["xbeam"]
ybeam = observations_pickle["ybeam"]
alpha_angle_obs = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \
for pred in mm_predictions])
spot_pred_x_mm = flex.double(
[pred[0] - xbeam for pred in mm_predictions])
spot_pred_y_mm = flex.double(
[pred[1] - ybeam for pred in mm_predictions])
#Polarization correction
wavelength = observations_pickle["wavelength"]
if iparams.flag_LP_correction:
fx = 1 - iparams.polarization_horizontal_fraction
fy = 1 - fx
if fx > 1.0 or fx < 0:
print 'Horizontal polarization fraction is not correct. The value must be >= 0 and <= 1'
print 'No polarization correction. Continue with post-refinement'
else:
phi_angle_obs = flex.double([math.atan2(pred[1]-ybeam, pred[0]-xbeam) \
for pred in mm_predictions])
bragg_angle_obs = observations.two_theta(wavelength).data()
P = ((fx*((flex.sin(phi_angle_obs)**2)+((flex.cos(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2)))+\
(fy*((flex.cos(phi_angle_obs)**2)+((flex.sin(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2))))
I_prime = observations.data() / P
sigI_prime = observations.sigmas() / P
observations = observations.customized_copy(
data=flex.double(I_prime), sigmas=flex.double(sigI_prime))
inputs = observations, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, \
detector_distance_mm, identified_isoform, \
mapped_predictions, xbeam, ybeam
return inputs, 'OK'