def postrefine_by_frame_mproc(args):
frame_no, pickle_result, iparams, miller_array_ref, avg_mode = args
prh = postref_handler()
pres, txt_out = prh.postrefine_by_frame(frame_no, pickle_result, iparams, miller_array_ref, avg_mode)
#overwrite to the original frame_no file.
if pres is not None:
pickle.dump(pres, open(iparams.run_no+'/pickles/'+str(pres.frame_no)+".o","wb"),pickle.HIGHEST_PROTOCOL)
def client():
result = []
prh = postref_handler()
while True:
comm.send(rank, dest=0)
msg = comm.recv(source=0)
if str(msg) == 'endrun': break
#receive contents for processing
activity, act_params = msg
if activity == "scale":
frame_files, iparams = act_params
for frame_index, frame_file in enumerate(frame_files):
pres, _ = prh.scale_frame_by_mean_I(frame_index, frame_file,
iparams, 0, 'average')
result.append(pres)
elif activity == "pre_merge":
frame_results, iparams, avg_mode = act_params
its = intensities_scaler()
prep_output = its.prepare_output(frame_results, iparams, avg_mode)
result.append(prep_output)
elif activity == "merge":
batch_prep, iparams, avg_mode = act_params
its = intensities_scaler()
mdh, _, txt_out_rejection = its.calc_avg_I_cpp(
batch_prep, iparams, avg_mode)
result.append([mdh, txt_out_rejection])
elif activity == "postref":
frame_no, frame_file, iparams, miller_array_ref, pres_in, avg_mode = act_params
pres, _ = prh.postrefine_by_frame(frame_no, frame_file, iparams,
miller_array_ref, pres_in,
avg_mode)
result.append(pres)
return result
def postrefine_by_frame_mproc(args):
from prime.postrefine import postref_handler
prh = postref_handler()
frame_no, frame_file, iparams, miller_array_ref, pres_in, avg_mode = args
pres = prh.postrefine_by_frame(frame_no, frame_file, iparams,
miller_array_ref, pres_in, avg_mode)
return pres
def client():
result = []
while True:
comm.send(rank, dest=0)
msg = comm.recv(source=0)
if str(msg) == "endrun":
break
# receive contents for processing
activity, act_params = msg
if activity == "scale":
frame_files, iparams = act_params
from prime.postrefine import postref_handler
prh = postref_handler()
for frame_index, frame_file in enumerate(frame_files):
pres, _ = prh.scale_frame_by_mean_I(frame_index, frame_file,
iparams, 0, "average")
result.append(pres)
if activity == "pre_merge":
frame_results, iparams = act_params
its = intensities_scaler()
prep_output = its.prepare_output(frame_results, iparams, "average")
result.append(prep_output)
if activity == "merge":
batch_prep, iparams = act_params
its = intensities_scaler()
mdh, _, txt_out_rejection = its.calc_avg_I_cpp(
batch_prep, iparams, "average")
result.append([mdh, txt_out_rejection])
return result
def determine_mean_I_mproc(args):
from prime.postrefine import postref_handler
prh = postref_handler()
frame_file, iparams, avg_mode = args
mean_I = prh.calc_mean_intensity(frame_file, iparams, avg_mode)
return mean_I
def scale_frame_by_mean_I_mproc(args):
from prime.postrefine import postref_handler
prh = postref_handler()
frame_no, frame_file, iparams, mean_of_mean_I, avg_mode = args
pres = prh.scale_frame_by_mean_I(frame_no, frame_file, iparams,
mean_of_mean_I, avg_mode)
return pres
def determine_mean_I_mproc(args):
frame_no, pickle_filename, iparams = args
prh = postref_handler()
mean_I, txt_out = prh.calc_mean_intensity(pickle_filename, iparams)
print frame_no, pickle_filename, mean_I
if mean_I is not None:
pickle.dump(mean_I, open(iparams.run_no+'/pickles/'+str(frame_no)+".o","wb"),pickle.HIGHEST_PROTOCOL)
def get_observations(self, pickle_filename, iparams): prh = postref_handler() #modify reflection filters iparams.merge.d_min = iparams.isoform_cluster.d_min iparams.merge.d_max = iparams.isoform_cluster.d_max iparams.merge.sigma_min = iparams.isoform_cluster.sigma_min pres, txt_out = prh.scale_frame_by_mean_I(-1, pickle_filename, iparams, 0, 'average') return pres.get_full_observations().merge_equivalents().array() if pres else None
def scale_frame_by_mean_I_mproc(args):
frame_no, pickle_filename, iparams, mm_I = args
prh = postref_handler()
pres, txt_out = prh.scale_frame_by_mean_I(frame_no, pickle_filename,
iparams, mm_I)
pickle.dump(
pres, open(iparams.run_no + '/pickles/' + str(frame_no) + ".o", "wb"),
pickle.HIGHEST_PROTOCOL)
def postrefine_by_frame_mproc(frame_no, frame_files, iparams, miller_array_ref, pres_results, avg_mode):
from prime.postrefine import postref_handler
prh = postref_handler()
if len(pres_results) == 0:
pres_in = None
else:
pres_in = pres_results[frame_no]
pres = prh.postrefine_by_frame(frame_no, frame_files[frame_no], iparams, miller_array_ref, pres_in, avg_mode)
return pres
def determine_mean_I_mproc(args):
frame_no, pickle_filename, iparams = args
prh = postref_handler()
mean_I, txt_out = prh.calc_mean_intensity(pickle_filename, iparams)
print frame_no, pickle_filename, mean_I
if mean_I is not None:
pickle.dump(
mean_I,
open(iparams.run_no + '/pickles/' + str(frame_no) + ".o", "wb"),
pickle.HIGHEST_PROTOCOL)
def get_observations(self, pickle_filename, iparams):
main_obs_pickle = pickle.load(open(pickle_filename,"rb"))
prh = postref_handler()
avg_mode = 'average'
try:
inputs, txt_org = prh.organize_input(main_obs_pickle, iparams, avg_mode, pickle_filename=pickle_filename)
main_obs = inputs[0]
except Exception:
print 'Error reading input pickle.'
return None
main_asu = main_obs.map_to_asu().merge_equivalents().array()
#get other indexing alternatives
if iparams.indexing_ambiguity.mode == 'Forced':
#generate other basis format according to the list
alternates = self.generate_forced_reindex_sets(main_asu, iparams.indexing_ambiguity.assigned_basis)
else:
alternates = self.generate_reindex_sets(main_asu)
return alternates
def get_observations(self, pickle_filename, iparams):
main_obs_pickle = pickle.load(open(pickle_filename,"rb"))
prh = postref_handler()
avg_mode = 'average'
try:
inputs, txt_org = prh.organize_input(main_obs_pickle, iparams, avg_mode, pickle_filename=pickle_filename)
main_obs = inputs[0]
except Exception:
print 'Error reading input pickle.'
return None
main_asu = main_obs.map_to_asu().merge_equivalents().array()
#get other indexing alternatives
if iparams.indexing_ambiguity.mode == 'Forced':
#generate other basis format according to the list
alternates = self.generate_forced_reindex_sets(main_asu, iparams.indexing_ambiguity.assigned_basis)
else:
alternates = self.generate_reindex_sets(main_asu)
return alternates
def run_by_params(self, iparams, avg_mode='average'):
#read all result pickles
try:
DIR = iparams.run_no+'/pickles/'
pickle_results = [pickle.load(open(DIR+fname, "rb")) for fname in os.listdir(DIR)]
n_results = len(pickle_results)
#for the last cycle merging with weak anomalous flag on
#fetch the original observations from the integration pickles
if iparams.flag_weak_anomalous and iparams.target_anomalous_flag and avg_mode=='final':
prh = postref_handler()
for pres in pickle_results:
if pres is not None:
observations_pickle = pickle.load(open(pres.pickle_filename,'rb'))
inputs, _ = prh.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pres.pickle_filename)
observations_original, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
two_theta = observations_original.two_theta(wavelength=pres.wavelength).data()
ph = partiality_handler()
partiality_set, delta_xy_set, rs_set, rh_set = ph.calc_partiality_anisotropy_set(
pres.unit_cell, pres.rotx, pres.roty, observations_original.indices(),
pres.ry, pres.rz, pres.r0, pres.re, pres.voigt_nu,
two_theta, alpha_angle_obs, pres.wavelength, pres.crystal_orientation,
spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm,
iparams.partiality_model, iparams.flag_beam_divergence)
observations, alt_hkl = prh.get_observations_non_polar(observations_original, pres.pickle_filename, iparams)
pres.observations = observations
pres.observations_original = observations_original
pres.partiality = partiality_set
pres.rs_set = rs_set
pres.rh_set = rh_set
pres.mapped_predictions = mapped_predictions
except Exception:
print "Error reading input pickles."
print "Check if prime.run, prime.genref, or prime.postrefine was called prior to merge."
exit()
from prime.postrefine import prepare_output
prep_output = prepare_output(pickle_results, iparams, avg_mode)
txt_merge_mean = 'prime.merge\n'
if prep_output is not None:
#grab linear lists
cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, I_obs_all_sort, \
sigI_obs_all_sort, G_all_sort, B_all_sort, p_all_sort, rs_all_sort, wavelength_all_sort, \
sin_sq_all_sort, SE_all_sort, uc_mean, wavelength_mean, pickle_filename_all_sort, txt_prep_out = prep_output
#start averaging
from prime.postrefine import calc_avg_I_cpp
calc_average_I_result = calc_avg_I_cpp(cn_group, group_id_list, miller_indices_all_sort,
miller_indices_ori_all_sort, I_obs_all_sort,
sigI_obs_all_sort, G_all_sort, B_all_sort, p_all_sort,
rs_all_sort, wavelength_all_sort, sin_sq_all_sort,
SE_all_sort, avg_mode, iparams, pickle_filename_all_sort)
#grab average results
miller_index, I_merge, sigI_merge, stats, I_two_halves_tuple, txt_obs_out, txt_reject_out = calc_average_I_result
I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l, I_odd_l = I_two_halves_tuple
#remove stat items with nan
r_meas_w_top, r_meas_w_btm, r_meas_top, r_meas_btm, multiplicity = stats
sel = flex.bool([math.isnan(r_top) or math.isinf(r_top)\
or math.isnan(r_btm) or math.isinf(r_btm) for r_top, r_btm in zip(r_meas_w_top, r_meas_w_btm)])
inverse_sel = ~sel
inverse_isel = inverse_sel.iselection()
miller_index = miller_index.select(inverse_isel)
I_merge = I_merge.select(inverse_isel)
sigI_merge = sigI_merge.select(inverse_isel)
I_even = I_even.select(inverse_isel)
I_odd = I_odd.select(inverse_isel)
I_even_h = I_even_h.select(inverse_isel)
I_odd_h = I_odd_h.select(inverse_isel)
I_even_k = I_even_k.select(inverse_isel)
I_odd_k = I_odd_k.select(inverse_isel)
I_even_l = I_even_l.select(inverse_isel)
I_odd_l = I_odd_l.select(inverse_isel)
stat_all = (r_meas_w_top.select(inverse_isel), r_meas_w_btm.select(inverse_isel), \
r_meas_top.select(inverse_isel), r_meas_btm.select(inverse_isel), multiplicity.select(inverse_isel))
#write out rejected reflections
f = open(iparams.run_no+'/rejections.txt', 'w')
f.write(txt_reject_out)
f.close()
#get the latest no. of mtz file and write out the reflections
DIR = iparams.run_no+'/mtz/'
file_no = len([int(fname.split('.')[0]) for fname in os.listdir(DIR) if os.path.join(DIR, fname).endswith('mtz')])
from prime.postrefine import write_output
miller_array_ref, txt_merge_mean_table = write_output(miller_index,
I_merge, sigI_merge,
stat_all, (I_even, I_odd, I_even_h, I_odd_h,
I_even_k, I_odd_k, I_even_l, I_odd_l),
iparams, uc_mean,
wavelength_mean,
file_no,
avg_mode)
txt_merge_mean += txt_merge_mean_table + txt_prep_out
print txt_merge_mean
f = open(iparams.run_no+'/log.txt', 'a')
f.write(txt_merge_mean)
f.close()
def determine_mean_I_mproc(frame_no, frame_files, iparams, avg_mode): from prime.postrefine import postref_handler prh = postref_handler() mean_I = prh.calc_mean_intensity(frame_files[frame_no], iparams, avg_mode) return mean_I
def scale_frame_by_mean_I_mproc(args): frame_no, pickle_filename, iparams, mm_I = args prh = postref_handler() pres, txt_out = prh.scale_frame_by_mean_I(frame_no, pickle_filename, iparams, mm_I) pickle.dump(pres, open(iparams.run_no+'/pickles/'+str(frame_no)+".o","wb"),pickle.HIGHEST_PROTOCOL)
def scale_frame_by_mean_I_mproc(frame_no, frame_files, iparams, mean_of_mean_I, avg_mode): from prime.postrefine import postref_handler prh = postref_handler() pres = prh.scale_frame_by_mean_I(frame_no,frame_files[frame_no], iparams, mean_of_mean_I, avg_mode) return pres
def run_by_params(self, iparams, avg_mode='average'):
#read all result pickles
try:
DIR = iparams.run_no + '/pickles/'
pickle_results = [
pickle.load(open(DIR + fname, "rb"))
for fname in os.listdir(DIR)
]
n_results = len(pickle_results)
#for the last cycle merging with weak anomalous flag on
#fetch the original observations from the integration pickles
if iparams.flag_weak_anomalous and iparams.target_anomalous_flag and avg_mode == 'final':
prh = postref_handler()
for pres in pickle_results:
if pres is not None:
observations_pickle = pickle.load(
open(pres.pickle_filename, 'rb'))
inputs, _ = prh.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pres.pickle_filename)
observations_original, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
two_theta = observations_original.two_theta(
wavelength=pres.wavelength).data()
ph = partiality_handler()
partiality_set, delta_xy_set, rs_set, rh_set = ph.calc_partiality_anisotropy_set(
pres.unit_cell, pres.rotx, pres.roty,
observations_original.indices(), pres.ry, pres.rz,
pres.r0, pres.re, pres.voigt_nu, two_theta,
alpha_angle_obs, pres.wavelength,
pres.crystal_orientation, spot_pred_x_mm,
spot_pred_y_mm, detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
observations, alt_hkl = prh.get_observations_non_polar(
observations_original, pres.pickle_filename,
iparams)
pres.observations = observations
pres.observations_original = observations_original
pres.partiality = partiality_set
pres.rs_set = rs_set
pres.rh_set = rh_set
pres.mapped_predictions = mapped_predictions
except Exception:
print "Error reading input pickles."
print "Check if prime.run, prime.genref, or prime.postrefine was called prior to merge."
exit()
from prime.postrefine import prepare_output
prep_output = prepare_output(pickle_results, iparams, avg_mode)
txt_merge_mean = 'prime.merge\n'
if prep_output is not None:
#grab linear lists
cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, I_obs_all_sort, \
sigI_obs_all_sort, G_all_sort, B_all_sort, p_all_sort, rs_all_sort, wavelength_all_sort, \
sin_sq_all_sort, SE_all_sort, uc_mean, wavelength_mean, pickle_filename_all_sort, txt_prep_out = prep_output
#start averaging
from prime.postrefine import calc_avg_I_cpp
calc_average_I_result = calc_avg_I_cpp(
cn_group, group_id_list, miller_indices_all_sort,
miller_indices_ori_all_sort, I_obs_all_sort, sigI_obs_all_sort,
G_all_sort, B_all_sort, p_all_sort, rs_all_sort,
wavelength_all_sort, sin_sq_all_sort, SE_all_sort, avg_mode,
iparams, pickle_filename_all_sort)
#grab average results
miller_index, I_merge, sigI_merge, stats, I_two_halves_tuple, txt_obs_out, txt_reject_out = calc_average_I_result
I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l, I_odd_l = I_two_halves_tuple
#remove stat items with nan
r_meas_w_top, r_meas_w_btm, r_meas_top, r_meas_btm, multiplicity = stats
sel = flex.bool([math.isnan(r_top) or math.isinf(r_top)\
or math.isnan(r_btm) or math.isinf(r_btm) for r_top, r_btm in zip(r_meas_w_top, r_meas_w_btm)])
inverse_sel = ~sel
inverse_isel = inverse_sel.iselection()
miller_index = miller_index.select(inverse_isel)
I_merge = I_merge.select(inverse_isel)
sigI_merge = sigI_merge.select(inverse_isel)
I_even = I_even.select(inverse_isel)
I_odd = I_odd.select(inverse_isel)
I_even_h = I_even_h.select(inverse_isel)
I_odd_h = I_odd_h.select(inverse_isel)
I_even_k = I_even_k.select(inverse_isel)
I_odd_k = I_odd_k.select(inverse_isel)
I_even_l = I_even_l.select(inverse_isel)
I_odd_l = I_odd_l.select(inverse_isel)
stat_all = (r_meas_w_top.select(inverse_isel), r_meas_w_btm.select(inverse_isel), \
r_meas_top.select(inverse_isel), r_meas_btm.select(inverse_isel), multiplicity.select(inverse_isel))
#write out rejected reflections
f = open(iparams.run_no + '/rejections.txt', 'w')
f.write(txt_reject_out)
f.close()
#get the latest no. of mtz file and write out the reflections
DIR = iparams.run_no + '/mtz/'
file_no = len([
int(fname.split('.')[0]) for fname in os.listdir(DIR)
if os.path.join(DIR, fname).endswith('mtz')
])
from prime.postrefine import write_output
miller_array_ref, txt_merge_mean_table = write_output(
miller_index, I_merge, sigI_merge, stat_all,
(I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l,
I_odd_l), iparams, uc_mean, wavelength_mean, file_no,
avg_mode)
txt_merge_mean += txt_merge_mean_table + txt_prep_out
print txt_merge_mean
f = open(iparams.run_no + '/log.txt', 'a')
f.write(txt_merge_mean)
f.close()
