def postrefine_by_frame_mproc(args): frame_no, pickle_result, iparams, miller_array_ref, avg_mode = args prh = postref_handler() pres, txt_out = prh.postrefine_by_frame(frame_no, pickle_result, iparams, miller_array_ref, avg_mode) #overwrite to the original frame_no file. if pres is not None: pickle.dump(pres, open(iparams.run_no+'/pickles/'+str(pres.frame_no)+".o","wb"),pickle.HIGHEST_PROTOCOL)
def client(): result = [] prh = postref_handler() while True: comm.send(rank, dest=0) msg = comm.recv(source=0) if str(msg) == 'endrun': break #receive contents for processing activity, act_params = msg if activity == "scale": frame_files, iparams = act_params for frame_index, frame_file in enumerate(frame_files): pres, _ = prh.scale_frame_by_mean_I(frame_index, frame_file, iparams, 0, 'average') result.append(pres) elif activity == "pre_merge": frame_results, iparams, avg_mode = act_params its = intensities_scaler() prep_output = its.prepare_output(frame_results, iparams, avg_mode) result.append(prep_output) elif activity == "merge": batch_prep, iparams, avg_mode = act_params its = intensities_scaler() mdh, _, txt_out_rejection = its.calc_avg_I_cpp( batch_prep, iparams, avg_mode) result.append([mdh, txt_out_rejection]) elif activity == "postref": frame_no, frame_file, iparams, miller_array_ref, pres_in, avg_mode = act_params pres, _ = prh.postrefine_by_frame(frame_no, frame_file, iparams, miller_array_ref, pres_in, avg_mode) result.append(pres) return result
def postrefine_by_frame_mproc(args): from prime.postrefine import postref_handler prh = postref_handler() frame_no, frame_file, iparams, miller_array_ref, pres_in, avg_mode = args pres = prh.postrefine_by_frame(frame_no, frame_file, iparams, miller_array_ref, pres_in, avg_mode) return pres
def client(): result = [] while True: comm.send(rank, dest=0) msg = comm.recv(source=0) if str(msg) == "endrun": break # receive contents for processing activity, act_params = msg if activity == "scale": frame_files, iparams = act_params from prime.postrefine import postref_handler prh = postref_handler() for frame_index, frame_file in enumerate(frame_files): pres, _ = prh.scale_frame_by_mean_I(frame_index, frame_file, iparams, 0, "average") result.append(pres) if activity == "pre_merge": frame_results, iparams = act_params its = intensities_scaler() prep_output = its.prepare_output(frame_results, iparams, "average") result.append(prep_output) if activity == "merge": batch_prep, iparams = act_params its = intensities_scaler() mdh, _, txt_out_rejection = its.calc_avg_I_cpp( batch_prep, iparams, "average") result.append([mdh, txt_out_rejection]) return result
def determine_mean_I_mproc(args): from prime.postrefine import postref_handler prh = postref_handler() frame_file, iparams, avg_mode = args mean_I = prh.calc_mean_intensity(frame_file, iparams, avg_mode) return mean_I
def scale_frame_by_mean_I_mproc(args): from prime.postrefine import postref_handler prh = postref_handler() frame_no, frame_file, iparams, mean_of_mean_I, avg_mode = args pres = prh.scale_frame_by_mean_I(frame_no, frame_file, iparams, mean_of_mean_I, avg_mode) return pres
def determine_mean_I_mproc(args): frame_no, pickle_filename, iparams = args prh = postref_handler() mean_I, txt_out = prh.calc_mean_intensity(pickle_filename, iparams) print frame_no, pickle_filename, mean_I if mean_I is not None: pickle.dump(mean_I, open(iparams.run_no+'/pickles/'+str(frame_no)+".o","wb"),pickle.HIGHEST_PROTOCOL)
def get_observations(self, pickle_filename, iparams): prh = postref_handler() #modify reflection filters iparams.merge.d_min = iparams.isoform_cluster.d_min iparams.merge.d_max = iparams.isoform_cluster.d_max iparams.merge.sigma_min = iparams.isoform_cluster.sigma_min pres, txt_out = prh.scale_frame_by_mean_I(-1, pickle_filename, iparams, 0, 'average') return pres.get_full_observations().merge_equivalents().array() if pres else None
def scale_frame_by_mean_I_mproc(args): frame_no, pickle_filename, iparams, mm_I = args prh = postref_handler() pres, txt_out = prh.scale_frame_by_mean_I(frame_no, pickle_filename, iparams, mm_I) pickle.dump( pres, open(iparams.run_no + '/pickles/' + str(frame_no) + ".o", "wb"), pickle.HIGHEST_PROTOCOL)
def postrefine_by_frame_mproc(frame_no, frame_files, iparams, miller_array_ref, pres_results, avg_mode): from prime.postrefine import postref_handler prh = postref_handler() if len(pres_results) == 0: pres_in = None else: pres_in = pres_results[frame_no] pres = prh.postrefine_by_frame(frame_no, frame_files[frame_no], iparams, miller_array_ref, pres_in, avg_mode) return pres
def determine_mean_I_mproc(args): frame_no, pickle_filename, iparams = args prh = postref_handler() mean_I, txt_out = prh.calc_mean_intensity(pickle_filename, iparams) print frame_no, pickle_filename, mean_I if mean_I is not None: pickle.dump( mean_I, open(iparams.run_no + '/pickles/' + str(frame_no) + ".o", "wb"), pickle.HIGHEST_PROTOCOL)
def get_observations(self, pickle_filename, iparams): main_obs_pickle = pickle.load(open(pickle_filename,"rb")) prh = postref_handler() avg_mode = 'average' try: inputs, txt_org = prh.organize_input(main_obs_pickle, iparams, avg_mode, pickle_filename=pickle_filename) main_obs = inputs[0] except Exception: print 'Error reading input pickle.' return None main_asu = main_obs.map_to_asu().merge_equivalents().array() #get other indexing alternatives if iparams.indexing_ambiguity.mode == 'Forced': #generate other basis format according to the list alternates = self.generate_forced_reindex_sets(main_asu, iparams.indexing_ambiguity.assigned_basis) else: alternates = self.generate_reindex_sets(main_asu) return alternates
def run_by_params(self, iparams, avg_mode='average'): #read all result pickles try: DIR = iparams.run_no+'/pickles/' pickle_results = [pickle.load(open(DIR+fname, "rb")) for fname in os.listdir(DIR)] n_results = len(pickle_results) #for the last cycle merging with weak anomalous flag on #fetch the original observations from the integration pickles if iparams.flag_weak_anomalous and iparams.target_anomalous_flag and avg_mode=='final': prh = postref_handler() for pres in pickle_results: if pres is not None: observations_pickle = pickle.load(open(pres.pickle_filename,'rb')) inputs, _ = prh.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pres.pickle_filename) observations_original, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs two_theta = observations_original.two_theta(wavelength=pres.wavelength).data() ph = partiality_handler() partiality_set, delta_xy_set, rs_set, rh_set = ph.calc_partiality_anisotropy_set( pres.unit_cell, pres.rotx, pres.roty, observations_original.indices(), pres.ry, pres.rz, pres.r0, pres.re, pres.voigt_nu, two_theta, alpha_angle_obs, pres.wavelength, pres.crystal_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) observations, alt_hkl = prh.get_observations_non_polar(observations_original, pres.pickle_filename, iparams) pres.observations = observations pres.observations_original = observations_original pres.partiality = partiality_set pres.rs_set = rs_set pres.rh_set = rh_set pres.mapped_predictions = mapped_predictions except Exception: print "Error reading input pickles." print "Check if prime.run, prime.genref, or prime.postrefine was called prior to merge." exit() from prime.postrefine import prepare_output prep_output = prepare_output(pickle_results, iparams, avg_mode) txt_merge_mean = 'prime.merge\n' if prep_output is not None: #grab linear lists cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, I_obs_all_sort, \ sigI_obs_all_sort, G_all_sort, B_all_sort, p_all_sort, rs_all_sort, wavelength_all_sort, \ sin_sq_all_sort, SE_all_sort, uc_mean, wavelength_mean, pickle_filename_all_sort, txt_prep_out = prep_output #start averaging from prime.postrefine import calc_avg_I_cpp calc_average_I_result = calc_avg_I_cpp(cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, I_obs_all_sort, sigI_obs_all_sort, G_all_sort, B_all_sort, p_all_sort, rs_all_sort, wavelength_all_sort, sin_sq_all_sort, SE_all_sort, avg_mode, iparams, pickle_filename_all_sort) #grab average results miller_index, I_merge, sigI_merge, stats, I_two_halves_tuple, txt_obs_out, txt_reject_out = calc_average_I_result I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l, I_odd_l = I_two_halves_tuple #remove stat items with nan r_meas_w_top, r_meas_w_btm, r_meas_top, r_meas_btm, multiplicity = stats sel = flex.bool([math.isnan(r_top) or math.isinf(r_top)\ or math.isnan(r_btm) or math.isinf(r_btm) for r_top, r_btm in zip(r_meas_w_top, r_meas_w_btm)]) inverse_sel = ~sel inverse_isel = inverse_sel.iselection() miller_index = miller_index.select(inverse_isel) I_merge = I_merge.select(inverse_isel) sigI_merge = sigI_merge.select(inverse_isel) I_even = I_even.select(inverse_isel) I_odd = I_odd.select(inverse_isel) I_even_h = I_even_h.select(inverse_isel) I_odd_h = I_odd_h.select(inverse_isel) I_even_k = I_even_k.select(inverse_isel) I_odd_k = I_odd_k.select(inverse_isel) I_even_l = I_even_l.select(inverse_isel) I_odd_l = I_odd_l.select(inverse_isel) stat_all = (r_meas_w_top.select(inverse_isel), r_meas_w_btm.select(inverse_isel), \ r_meas_top.select(inverse_isel), r_meas_btm.select(inverse_isel), multiplicity.select(inverse_isel)) #write out rejected reflections f = open(iparams.run_no+'/rejections.txt', 'w') f.write(txt_reject_out) f.close() #get the latest no. of mtz file and write out the reflections DIR = iparams.run_no+'/mtz/' file_no = len([int(fname.split('.')[0]) for fname in os.listdir(DIR) if os.path.join(DIR, fname).endswith('mtz')]) from prime.postrefine import write_output miller_array_ref, txt_merge_mean_table = write_output(miller_index, I_merge, sigI_merge, stat_all, (I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l, I_odd_l), iparams, uc_mean, wavelength_mean, file_no, avg_mode) txt_merge_mean += txt_merge_mean_table + txt_prep_out print txt_merge_mean f = open(iparams.run_no+'/log.txt', 'a') f.write(txt_merge_mean) f.close()
def determine_mean_I_mproc(frame_no, frame_files, iparams, avg_mode): from prime.postrefine import postref_handler prh = postref_handler() mean_I = prh.calc_mean_intensity(frame_files[frame_no], iparams, avg_mode) return mean_I
def scale_frame_by_mean_I_mproc(args): frame_no, pickle_filename, iparams, mm_I = args prh = postref_handler() pres, txt_out = prh.scale_frame_by_mean_I(frame_no, pickle_filename, iparams, mm_I) pickle.dump(pres, open(iparams.run_no+'/pickles/'+str(frame_no)+".o","wb"),pickle.HIGHEST_PROTOCOL)
def scale_frame_by_mean_I_mproc(frame_no, frame_files, iparams, mean_of_mean_I, avg_mode): from prime.postrefine import postref_handler prh = postref_handler() pres = prh.scale_frame_by_mean_I(frame_no,frame_files[frame_no], iparams, mean_of_mean_I, avg_mode) return pres
def run_by_params(self, iparams, avg_mode='average'): #read all result pickles try: DIR = iparams.run_no + '/pickles/' pickle_results = [ pickle.load(open(DIR + fname, "rb")) for fname in os.listdir(DIR) ] n_results = len(pickle_results) #for the last cycle merging with weak anomalous flag on #fetch the original observations from the integration pickles if iparams.flag_weak_anomalous and iparams.target_anomalous_flag and avg_mode == 'final': prh = postref_handler() for pres in pickle_results: if pres is not None: observations_pickle = pickle.load( open(pres.pickle_filename, 'rb')) inputs, _ = prh.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pres.pickle_filename) observations_original, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs two_theta = observations_original.two_theta( wavelength=pres.wavelength).data() ph = partiality_handler() partiality_set, delta_xy_set, rs_set, rh_set = ph.calc_partiality_anisotropy_set( pres.unit_cell, pres.rotx, pres.roty, observations_original.indices(), pres.ry, pres.rz, pres.r0, pres.re, pres.voigt_nu, two_theta, alpha_angle_obs, pres.wavelength, pres.crystal_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) observations, alt_hkl = prh.get_observations_non_polar( observations_original, pres.pickle_filename, iparams) pres.observations = observations pres.observations_original = observations_original pres.partiality = partiality_set pres.rs_set = rs_set pres.rh_set = rh_set pres.mapped_predictions = mapped_predictions except Exception: print "Error reading input pickles." print "Check if prime.run, prime.genref, or prime.postrefine was called prior to merge." exit() from prime.postrefine import prepare_output prep_output = prepare_output(pickle_results, iparams, avg_mode) txt_merge_mean = 'prime.merge\n' if prep_output is not None: #grab linear lists cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, I_obs_all_sort, \ sigI_obs_all_sort, G_all_sort, B_all_sort, p_all_sort, rs_all_sort, wavelength_all_sort, \ sin_sq_all_sort, SE_all_sort, uc_mean, wavelength_mean, pickle_filename_all_sort, txt_prep_out = prep_output #start averaging from prime.postrefine import calc_avg_I_cpp calc_average_I_result = calc_avg_I_cpp( cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, I_obs_all_sort, sigI_obs_all_sort, G_all_sort, B_all_sort, p_all_sort, rs_all_sort, wavelength_all_sort, sin_sq_all_sort, SE_all_sort, avg_mode, iparams, pickle_filename_all_sort) #grab average results miller_index, I_merge, sigI_merge, stats, I_two_halves_tuple, txt_obs_out, txt_reject_out = calc_average_I_result I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l, I_odd_l = I_two_halves_tuple #remove stat items with nan r_meas_w_top, r_meas_w_btm, r_meas_top, r_meas_btm, multiplicity = stats sel = flex.bool([math.isnan(r_top) or math.isinf(r_top)\ or math.isnan(r_btm) or math.isinf(r_btm) for r_top, r_btm in zip(r_meas_w_top, r_meas_w_btm)]) inverse_sel = ~sel inverse_isel = inverse_sel.iselection() miller_index = miller_index.select(inverse_isel) I_merge = I_merge.select(inverse_isel) sigI_merge = sigI_merge.select(inverse_isel) I_even = I_even.select(inverse_isel) I_odd = I_odd.select(inverse_isel) I_even_h = I_even_h.select(inverse_isel) I_odd_h = I_odd_h.select(inverse_isel) I_even_k = I_even_k.select(inverse_isel) I_odd_k = I_odd_k.select(inverse_isel) I_even_l = I_even_l.select(inverse_isel) I_odd_l = I_odd_l.select(inverse_isel) stat_all = (r_meas_w_top.select(inverse_isel), r_meas_w_btm.select(inverse_isel), \ r_meas_top.select(inverse_isel), r_meas_btm.select(inverse_isel), multiplicity.select(inverse_isel)) #write out rejected reflections f = open(iparams.run_no + '/rejections.txt', 'w') f.write(txt_reject_out) f.close() #get the latest no. of mtz file and write out the reflections DIR = iparams.run_no + '/mtz/' file_no = len([ int(fname.split('.')[0]) for fname in os.listdir(DIR) if os.path.join(DIR, fname).endswith('mtz') ]) from prime.postrefine import write_output miller_array_ref, txt_merge_mean_table = write_output( miller_index, I_merge, sigI_merge, stat_all, (I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l, I_odd_l), iparams, uc_mean, wavelength_mean, file_no, avg_mode) txt_merge_mean += txt_merge_mean_table + txt_prep_out print txt_merge_mean f = open(iparams.run_no + '/log.txt', 'a') f.write(txt_merge_mean) f.close()