def merge_observations_handler(self,
obs_hdlr_ref,
obs_hdlr,
r_final_cutoff=1.0):
"""
Take the reference (full observations) and merge with the given
observations (need to correct for scale and partiality).
"""
if obs_hdlr.r_final < r_final_cutoff:
obs_ref = obs_hdlr_ref.observations.deep_copy()
obs = obs_hdlr.calc_full_observations()
miller_indices_all = obs_ref.indices()
I_all = obs_ref.data()
sigI_all = obs_ref.sigmas()
miller_indices_all.extend(obs.indices())
I_all.extend(obs.data())
sigI_all.extend(obs.sigmas())
uc_mean, uc_med, uc_std = self.calc_mean_unit_cell(
[obs_hdlr_ref, obs_hdlr])
crystal_symmetry = crystal.symmetry(
unit_cell=(uc_mean[0], uc_mean[1], uc_mean[2], uc_mean[3],
uc_mean[4], uc_mean[5]),
space_group_symbol=self.iparams.target_space_group)
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=miller_indices_all,
anomalous_flag=self.iparams.target_anomalous_flag)
miller_array_all = miller_set.array(
data=I_all,
sigmas=sigI_all).set_observation_type_xray_intensity()
me_obj = miller_array_all.merge_equivalents()
#build newly merged observations_handler
observations_hdlr_merge = observations_handler(self.iparams)
observations_hdlr_merge.init_params_from_merge(me_obj.array())
txt_out = ' {0:40} ==> MERGED RES:{1:5.2f} NREFL:{2:5d} R_MEAS:{3:6.2f} R_MERGE:{4:6.2f}'.format(
observations_hdlr_merge.pickle_filename_only,
observations_hdlr_merge.observations.d_min(),
len(observations_hdlr_merge.observations.data()),
me_obj.r_meas(), me_obj.r_merge())
else:
observations_hdlr_merge = observations_handler(self.iparams)
observations_hdlr_merge.copy_from(obs_hdlr_ref)
txt_out = ' {0:40} ==> NOT MERGED (Rfin too low)'.format(
observations_hdlr_merge.pickle_filename_only)
return observations_hdlr_merge, txt_out
def scale_with_reference(self,
observations_hdlr_ref,
filename=None,
observations_hdlr=None,
use_binning=False):
"""
Scale each emage to the reference.
"""
if True:
if filename is not None:
observations_hdlr = observations_handler(self.iparams)
observations_hdlr.init_params_from_file(filename)
#find G and B-tensor for high resolution reflections
lsqr_hdlr = leastsqr_handler(self.iparams)
lsqr_hdlr.optimize_scalefactors(observations_hdlr_ref,
observations_hdlr,
use_binning=use_binning)
#find G low resolution
obs_ref_lowres = observations_hdlr_ref.observations.resolution_filter(
d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
obs_lowres = observations_hdlr.observations.resolution_filter(
d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
obs_lowres = obs_lowres.select(
flex.bool(
np.absolute(
(obs_lowres.data() - np.mean(obs_lowres.data())) /
np.std(obs_lowres.data())) < self.sigma_I_cutoff))
try:
G_lowres = obs_lowres.mean() / obs_ref_lowres.mean()
except Exception:
G_lowres = observations_hdlr.G
observations_hdlr.set_params(G_lowres=G_lowres)
txt_out = ' {0:40} ==> SCALED RES:{1:5.2f} NREFL:{2:5d} SG:{14:8} Glow:{15:6.1f} G:{3:6.1f} B:{4:6.1f} {5:6.1f} {6:6.1f} {7:6.1f} {8:6.1f} {9:6.1f} Rfin:{11:5.2f} CCini:{12:5.2f} CCfin:{13:5.2f}'.format(
observations_hdlr.pickle_filename_only,
observations_hdlr.observations.d_min(),
len(observations_hdlr.observations.data()),
observations_hdlr.G, observations_hdlr.b11,
observations_hdlr.b22, observations_hdlr.b33,
observations_hdlr.b12, observations_hdlr.b13,
observations_hdlr.b23, observations_hdlr.r_init,
observations_hdlr.r_final, observations_hdlr.cc_init,
observations_hdlr.cc_final,
observations_hdlr.observations.space_group_info(), G_lowres)
#except Exception:
# return None, ' {0:40} ==> SCALING FAILED'.format(observations_hdlr.pickle_filename_only)
return observations_hdlr, txt_out
def optimize_scalefactors(self, observations_hdlr_ref, observations_hdlr, use_binning=False):
"""
Keep other partiality parameters constant and refine only scale factors.
"""
observations_hdlr_sel = observations_handler(self.iparams)
observations_hdlr_sel.copy_from(observations_hdlr)
observations_hdlr_sel.resolution_filter(d_min=self.iparams.d_min, d_max=self.scale_0_d_max)
observations_hdlr_sel.outlier_filter(sigma_I_cutoff=self.sigma_I_cutoff)
refine_mode = 'scale_factor'
if observations_hdlr.postref_params is None:
a,b,c,alpha,beta,gamma = observations_hdlr.observations.unit_cell().parameters()
postref_params = [1,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0,self.iparams.gamma_e,a,b,c,alpha,beta,gamma]
else:
postref_params = observations_hdlr.postref_params[:]
if use_binning:
#use binning
obs_ref = observations_hdlr_ref.observations.deep_copy()
obs_ref = obs_ref.resolution_filter(d_min=self.iparams.d_min, d_max=self.scale_0_d_max)
binner = obs_ref.setup_binner(n_bins=30)
mean_I_ref = obs_ref.mean(
use_binning=True,
use_multiplicities=True).data[1:-1]
obs = observations_hdlr_sel.calc_full_observations()
obs.use_binning_of(obs_ref)
mean_I_obs = obs.mean(
use_binning=True,
use_multiplicities=True).data[1:-1]
stol_sq = obs_ref.sin_theta_over_lambda_sq()
stol_sq.use_binning_of(obs_ref)
mean_stol_sq = stol_sq.mean(
use_binning=True,
use_multiplicities=True).data[1:-1]
mean_I_ref_sel = flex.double()
mean_I_obs_sel = flex.double()
mean_stol_sq_sel = flex.double()
for i in range(len(mean_I_obs)):
if mean_I_ref[i] is not None and mean_I_obs[i] is not None and mean_stol_sq[i] is not None:
mean_I_ref_sel.append(mean_I_ref[i])
mean_I_obs_sel.append(mean_I_obs[i])
mean_stol_sq_sel.append(mean_stol_sq[i])
xinp = np.array([1,0])
xopt, cov_x, infodict, mesg, ier = optimize.leastsq(self.func_scale, xinp,
args=(mean_I_ref_sel, mean_I_obs_sel, mean_stol_sq_sel, None),
full_output=True)
G, biso = xopt
b11, b22, b33, b12, b13, b23 = (biso,biso,biso,0,0,0)
else:
#use common reflections
obs_hdlr_ref, obs_hdlr = observations_hdlr_ref.common_filter(observations_hdlr_sel)
I_r = obs_hdlr_ref.observations.data()
cs = observations_hdlr.observations_original.crystal_symmetry().space_group().crystal_system()
G, b11, b22, b33, b12, b13, b23, rotx, roty, ry, rz, r0, re, a, b, c, alpha, beta, gamma = observations_hdlr.postref_params
const_params = [rotx, roty, ry, rz, r0, re, a, b, c, alpha, beta, gamma]
b_tensor_indep = self.prep_input([b11, b22, b33, b12, b13, b23], cs)
xinp = np.append(G, b_tensor_indep)
xopt, cov_x, infodict, mesg, ier = optimize.leastsq(self.func, xinp,
args=(I_r, obs_hdlr, const_params, refine_mode),
full_output=True)
G = xopt[0]
b11, b22, b33, b12, b13, b23 = self.prep_output(xopt[1:], cs, output_type='b_tensor')
postref_params[:7] = [G,b11, b22, b33, b12, b13, b23]
observations_hdlr.set_params(postref_params=postref_params)
cc_init, cc_final, r_init, r_final, r_xy_init, r_xy_final, cc_iso_init, cc_iso_final = (0,0,0,0,0,0,0,0)
if True:
obs_ref, obs = observations_hdlr_ref.observations.common_sets(observations_hdlr.observations,
assert_is_similar_symmetry=False)
obs_ref, obs_full = observations_hdlr_ref.observations.common_sets(observations_hdlr.calc_full_observations(),
assert_is_similar_symmetry=False)
cc_init = obs_ref.correlation(other=obs, use_binning=False, assert_is_similar_symmetry=False).coefficient()
cc_final = obs_ref.correlation(other=obs_full, use_binning=False, assert_is_similar_symmetry=False).coefficient()
r_init = flex.sum((obs_ref.data()-obs.data())**2)/flex.sum(obs_ref.data()**2)
r_final = flex.sum((obs_ref.data()-obs_full.data())**2)/flex.sum(obs_ref.data()**2)
#except Exception:
# pass
observations_hdlr.set_params(stats=[cc_init, cc_final, r_init, r_final, r_xy_init, r_xy_final, cc_iso_init, cc_iso_final])
def merge_multi_observations_handler(self,
observations_hdlr_set,
flag_show_summary=False,
r_final_cutoff=1.0):
"""
Merge all observations_hdlr in the set
"""
miller_indices_all = flex.miller_index()
I_all = flex.double()
sigI_all = flex.double()
cn_good_observations = 0
observations_hdlr_selected_set = []
for i_frame in range(len(observations_hdlr_set)):
if observations_hdlr_set[i_frame].r_final < r_final_cutoff:
obs_full = observations_hdlr_set[
i_frame].calc_full_observations()
if obs_full is not None:
cn_good_observations += 1
observations_hdlr_selected_set.append(
observations_hdlr_set[i_frame])
miller_indices_all.extend(obs_full.indices())
I_all.extend(obs_full.data())
sigI_all.extend(obs_full.sigmas())
uc_mean, uc_med, uc_std = self.calc_mean_unit_cell(
observations_hdlr_selected_set)
try:
crystal_symmetry = crystal.symmetry(
unit_cell=(uc_mean[0], uc_mean[1], uc_mean[2], uc_mean[3],
uc_mean[4], uc_mean[5]),
space_group_symbol=self.iparams.target_space_group)
except Exception:
print uc_mean[0], uc_mean[1], uc_mean[2], uc_mean[3], uc_mean[
4], uc_mean[5]
uc_mean = self.iparams.target_unit_cell.parameters()
crystal_symmetry = crystal.symmetry(
unit_cell=(uc_mean[0], uc_mean[1], uc_mean[2], uc_mean[3],
uc_mean[4], uc_mean[5]),
space_group_symbol=self.iparams.target_space_group)
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=miller_indices_all,
anomalous_flag=self.iparams.target_anomalous_flag)
miller_array_all = miller_set.array(
data=I_all, sigmas=sigI_all).set_observation_type_xray_intensity()
if miller_array_all.size() == 0:
txt_out = ' FINAL MERGE FAILED'
return None, None, txt_out
#remove outlier reflections
perm = miller_array_all.sort_permutation(by_value="packed_indices")
miller_array_all_sort = miller_array_all.select(perm)
current_miller_index = flex.miller_index(
[miller_array_all_sort.indices()[0]])
group_id = 0
group_id_set = flex.int([0] * len(miller_array_all_sort.indices()))
for miller_index, i_seq in zip(
miller_array_all_sort.indices(),
range(len(miller_array_all_sort.indices()))):
if current_miller_index[0][0] == miller_index[
0] and current_miller_index[0][1] == miller_index[
1] and current_miller_index[0][2] == miller_index[2]:
group_id_set[i_seq] = group_id
else:
current_miller_index = flex.miller_index(
[miller_array_all_sort.indices()[i_seq]])
group_id += 1
group_id_set[i_seq] = group_id
miller_indices_all = flex.miller_index()
I_all = flex.double()
sigI_all = flex.double()
for group_id in range(max(group_id_set) + 1):
miller_array_group = miller_array_all_sort.select(
group_id_set == group_id)
mean_I_over_sigI = flex.mean(miller_array_group.data() /
miller_array_group.sigmas())
std_I_over_sigI = math.sqrt(mean_I_over_sigI)
#miller_array_group = miller_array_group.select(flex.bool(np.absolute((miller_array_group.data()-np.mean(miller_array_group.data()))/np.std(miller_array_group.data())) < self.sigma_I_cutoff))
miller_array_group = miller_array_group.select(
flex.bool((miller_array_group.data() /
miller_array_group.sigmas()) > mean_I_over_sigI +
(2 * std_I_over_sigI)))
miller_indices_all.extend(miller_array_group.indices())
I_all.extend(miller_array_group.data())
sigI_all.extend(miller_array_group.sigmas())
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=miller_indices_all,
anomalous_flag=self.iparams.target_anomalous_flag)
miller_array_all = miller_set.array(
data=I_all, sigmas=sigI_all).set_observation_type_xray_intensity()
miller_array_all = miller_array_all.resolution_filter(
d_min=self.iparams.d_min, d_max=self.iparams.d_max)
#merge
me_obj = miller_array_all.merge_equivalents()
miller_array_merge = me_obj.array().as_intensity_array()
if miller_array_merge.size() == 0:
txt_out = ' FINAL MERGE FAILED'
return None, None, txt_out
miller_array_merge.setup_binner(n_bins=self.iparams.n_bins)
#get asu_contents
asu_contents = {}
asu_volume = miller_array_merge.unit_cell().volume() / float(
miller_array_merge.space_group().order_z())
number_carbons = asu_volume / 18.0
asu_contents.setdefault('C', number_carbons)
#build newly merged observations_handler
observations_hdlr_merge = observations_handler(self.iparams)
observations_hdlr_merge.init_params_from_merge(miller_array_merge)
#write as mtz file
mtz_dataset_merge = miller_array_merge.as_mtz_dataset(
column_root_label="IOBS")
mtz_dataset_merge.mtz_object().write(file_name=self.iparams.run_no +
'/postref_merged.mtz')
txt_out = ' FINAL MERGE RES:{0:5.2f} NREFL:{1:5d} NIMG:{2:6d} R_MEAS:{3:6.2f} R_MERGE:{4:6.2f}'.format(
observations_hdlr_merge.observations.d_min(),
len(observations_hdlr_merge.observations.data()),
cn_good_observations, me_obj.r_meas(), me_obj.r_merge())
if flag_show_summary:
print 'SUMMARY: mean intensity (%6.0f images)' % (
cn_good_observations)
ma_mean_I = miller_array_merge.mean(use_binning=True)
ma_mean_I.show()
print ''
from mod_util import utility_handler
util_hdlr = utility_handler()
flag_hklisoin_found, miller_array_iso = util_hdlr.get_miller_array_from_mtz(
self.iparams.hklisoin)
if flag_hklisoin_found:
try:
print 'SUMMARY: CCiso'
ma_cc_iso = miller_array_merge.correlation(
miller_array_iso,
use_binning=True,
assert_is_similar_symmetry=False)
ma_cc_iso.show()
print ''
except Exception:
pass
print 'SUMMARY: Merging statistics'
me_obj.show_summary()
print ''
if self.iparams.target_anomalous_flag:
miller_array_all.setup_binner(n_bins=self.iparams.n_bins)
ma_cc_anom = miller_array_all.cc_anom(use_binning=True)
ma_cc_anom.show()
from mod_util import utility_handler
util_hdlr = utility_handler()
flag_hklisoin_found, miller_array_iso = util_hdlr.get_miller_array_from_mtz(
self.iparams.hklisoin)
miller_array_iso = miller_array_iso.as_intensity_array()
if self.iparams.target_anomalous_flag:
miller_array_iso = miller_array_iso.generate_bijvoet_mates()
observations_hdlr_iso = observations_handler(self.iparams)
observations_hdlr_iso.init_params_from_merge(miller_array_iso)
from mod_plot import plot_handler
plot_hdlr = plot_handler(self.iparams)
plot_hdlr.plot_wilson(observations_hdlr_iso,
observations_hdlr_selected_set)
return observations_hdlr_merge, observations_hdlr_selected_set, txt_out
def scale_0(self, filename):
"""
Perform 0th-order scaling.
"""
observations_hdlr = observations_handler(self.iparams)
observations_hdlr.init_params_from_file(filename)
"""
#get the miller_array_template
uc = self.iparams.target_unit_cell.parameters()
crystal_symmetry = crystal.symmetry(
unit_cell=(uc[0], uc[1], uc[2], uc[3], uc[4], uc[5]),
space_group_symbol=self.iparams.target_space_group
)
miller_array_complete = crystal_symmetry.build_miller_set(anomalous_flag=self.iparams.target_anomalous_flag,
d_min=self.iparams.d_min, d_max=self.iparams.d_max).array()
miller_array_complete = miller_array_complete.set_observation_type_xray_intensity()
miller_array_complete = miller_array_complete.customized_copy(data=flex.double([1e4]*len(miller_array_complete.indices())),
sigmas=flex.double([1e2]*len(miller_array_complete.indices())))
asu_contents = {}
asu_volume = miller_array_complete.unit_cell().volume()/float(miller_array_complete.space_group().order_z())
number_carbons = asu_volume/18.0
asu_contents.setdefault('C', number_carbons)
ftmpl = miller_array_complete.as_amplitude_array()
ftmpl.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(ftmpl, asu_contents, e_statistics=True)
G, B = (wp.wilson_intensity_scale_factor*1e3,wp.wilson_b)
stolsq = miller_array_complete.two_theta(observations_hdlr.wavelength).sin_theta_over_lambda_sq().data()
I_wp = miller_array_complete.data()/(G * flex.exp(-2 * B * stolsq))
sigI_wp = miller_array_complete.sigmas()/(G * flex.exp(-2 * B * stolsq))
miller_array_complete = miller_array_complete.customized_copy(data=I_wp, sigmas=sigI_wp)
miller_array_hires = miller_array_complete.resolution_filter(d_min=self.iparams.d_min, d_max=self.scale_0_d_max)
miller_array_lowres = miller_array_complete.resolution_filter(d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
obs_lowres = observations_hdlr.observations.resolution_filter(d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
G_lowres = obs_lowres.mean()/miller_array_lowres.mean()
observations_hdlr.set_params(G_lowres=G_lowres)
"""
from mod_util import utility_handler
util_hdlr = utility_handler()
flag_hklisoin_found, miller_array_iso = util_hdlr.get_miller_array_from_mtz(
self.iparams.hklisoin)
miller_array_complete = miller_array_iso.as_intensity_array()
miller_array_complete = miller_array_complete.customized_copy(
data=miller_array_complete.data(),
sigmas=miller_array_complete.sigmas())
if self.iparams.target_anomalous_flag:
miller_array_complete = miller_array_complete.generate_bijvoet_mates(
)
miller_array_hires = miller_array_complete.resolution_filter(
d_min=self.iparams.d_min, d_max=self.scale_0_d_max)
miller_array_lowres = miller_array_complete.resolution_filter(
d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
observations_hdlr_ref = observations_handler(self.iparams)
observations_hdlr_ref.init_params_from_merge(miller_array_hires)
G, B = (1, 0)
#caculate G and B-tensor for high resolution reflections
if True:
observations_full = observations_hdlr.calc_full_observations()
lsqr_hdlr = leastsqr_handler(self.iparams)
lsqr_hdlr.optimize_scalefactors(observations_hdlr_ref,
observations_hdlr,
use_binning=False)
#except Exception:
# return None, ' {0:40} ==> SCALING FAILED'.format(observations_hdlr.pickle_filename_only)
#calculate G for low-resolution
obs_lowres = observations_hdlr.observations.resolution_filter(
d_min=self.scale_0_d_max, d_max=self.iparams.d_max)
obs_lowres = obs_lowres.select(
flex.bool(
np.absolute((obs_lowres.data() - np.mean(obs_lowres.data())) /
np.std(obs_lowres.data())) < self.sigma_I_cutoff))
G_lowres = obs_lowres.mean() / miller_array_lowres.mean()
observations_hdlr.set_params(G_lowres=G_lowres)
txt_out = ' {0:40} ==> SCALED RES:{1:5.2f} NREFL:{2:5d} SG:{12:8} Glow:{15:6.1f} G:{3:6.1f} B:{4:6.1f} {5:6.1f} {6:6.1f} {7:6.1f} {8:6.1f} {9:6.1f} Rfin:{11:5.2f} Gwp:{13:6.2f} Bwp{14:6.1f}'.format(
observations_hdlr.pickle_filename_only,
observations_hdlr.observations.d_min(),
len(observations_hdlr.observations.data()), observations_hdlr.G,
observations_hdlr.b11, observations_hdlr.b22,
observations_hdlr.b33, observations_hdlr.b12,
observations_hdlr.b13, observations_hdlr.b23,
observations_hdlr.r_init, observations_hdlr.r_final,
observations_hdlr.observations.space_group_info(), G, B, G_lowres)
return observations_hdlr, txt_out