def calculate_r_mproc(args): frame_no, pickle_filename, index_basis, obs_main, obs_list = args idah = indamb_handler() r_set, txt_out = idah.calc_r(frame_no, pickle_filename, index_basis, obs_main, obs_list) print txt_out return pickle_filename, index_basis, r_set
def main(data, only_merohedral): indambh = indamb_handler() intFileList = read_pickles([data]) if intFileList: obsList = {} for intFileName in intFileList: intPickle = read_frame(intFileName) try: obs = intPickle['observations'][0] obsList[intFileName] = obs except Exception as e: print "Warning:", e pass for key, value in obsList.iteritems(): if only_merohedral: flag_all = False else: flag_all = True ops = indambh.generate_twin_operators(value, flag_all=flag_all) if ops: print os.path.basename( key ), '%6.1f,%6.1f,%6.1f,%6.1f,%6.1f,%6.1f' % value.unit_cell( ).parameters(), ' '.join( [op.operator.r().as_hkl() for op in ops]) else: print os.path.basename( key ), '%6.1f,%6.1f,%6.1f,%6.1f,%6.1f,%6.1f' % value.unit_cell( ).parameters(), 'Twining operators not found'
def solve_with_mtz_mproc(args): frame_no, pickle_filename, iparams, miller_array_ref = args idah = indamb_handler() index_basis, txt_out = idah.calc_cc(frame_no, pickle_filename, iparams, miller_array_ref) print txt_out return pickle_filename, index_basis
def should_terminate(self, iparams, pickle_filename): #if no indexing ambiguity problem detected and mode is not set to "Forced" idah = indamb_handler() alt_dict = idah.get_observations(pickle_filename, iparams) if len(alt_dict.keys()) == 1 and iparams.indexing_ambiguity.mode != 'Forced': return True if iparams.indexing_ambiguity.mode == 'Forced' and len(iparams.indexing_ambiguity.assigned_basis) == 0: return True return False
def should_terminate(self, iparams, pickle_filename): #if no indexing ambiguity problem detected and mode is not set to "Forced" idah = indamb_handler() alt_dict = idah.get_observations(pickle_filename, iparams) if len(alt_dict.keys()) == 1 and iparams.indexing_ambiguity.mode != 'Forced': return True if iparams.indexing_ambiguity.mode == 'Forced' and len(iparams.indexing_ambiguity.assigned_basis) == 0: return True return False
def run(self, args): #read inputs from prime.postrefine.mod_input import process_input, read_pickles iparams, txt_out_input = process_input(args) print txt_out_input f = open(iparams.run_no+'/log.txt', 'w') f.write(txt_out_input) f.close() #if solution pickle is given, return the file name if iparams.indexing_ambiguity.index_basis_in is not None: if iparams.indexing_ambiguity.index_basis_in.endswith('.pickle'): return iparams.indexing_ambiguity.index_basis_in, iparams #read all integration pickles frame_files = read_pickles(iparams.data) n_frames = len(frame_files) if n_frames == 0: print "No integration pickle found. Exit program." return None, iparams #exit if no problem if self.should_terminate(iparams, frame_files[0]): print "No indexing ambiguity problem. Set index_ambiguity.mode = Forced and assigned_basis = list of basis formats to solve pseudo-twinning problem." return None, iparams #continue with (Auto - alt>1, find solution), (Auto - alt>1, mtz) #(Forced - assigned_basis, mtz), (Forced - assigned_basis, find solution) #************************************************* #if mtz file is given, use it to solve the problem sol_fname = iparams.run_no+'/index_ambiguity/solution_pickle.pickle' if iparams.indexing_ambiguity.index_basis_in is not None: if iparams.indexing_ambiguity.index_basis_in.endswith('.mtz'): mxh = mx_handler() flag_ref_found, miller_array_ref = mxh.get_miller_array_from_reflection_file(iparams.indexing_ambiguity.index_basis_in) if flag_ref_found == False: print "Reference mtz file not found. Set indexing_ambiguity.index_basis_in = None to enable auto generate the solutions." return None, iparams else: frames = [(i, frame_files[i], iparams, miller_array_ref) for i in range(n_frames)] cc_results = pool_map( iterable=frames, func=solve_with_mtz_mproc, processes=iparams.n_processors) sol_pickle = {} for result in cc_results: pickle_filename, index_basis = result sol_pickle[pickle_filename] = index_basis pickle.dump(sol_pickle, open(sol_fname,"wb")) return sol_fname, iparams #************************************************* #solve with Brehm & Diederichs - sample size n_sample_frames then bootstrap the rest frames = [(i, frame_files[i], iparams) for i in random.sample(range(n_frames), iparams.indexing_ambiguity.n_sample_frames)] #get observations list print "Reading observations" alt_dict_results = pool_map( iterable=frames, func=get_obs_mproc, processes=iparams.n_processors) frame_dup_files = [] frame_keys = [] obs_list = [] for result in alt_dict_results: alt_dict, pickle_filename = result if alt_dict is not None: for key in alt_dict.keys(): frame_dup_files.append(pickle_filename) frame_keys.append(key) obs_list.append(alt_dict[key]) frames = [(i, frame_dup_files[i], frame_keys[i], obs_list[i], obs_list) for i in range(len(frame_dup_files))] #calculate r print "Calculating R" calc_r_results = pool_map( iterable=frames, func=calculate_r_mproc, processes=iparams.n_processors) frame_dup_files = [] frame_keys = [] r_matrix = [] for result in calc_r_results: if result is not None: pickle_filename, index_basis, r_set = result frame_dup_files.append(pickle_filename) frame_keys.append(index_basis) if len(r_matrix) == 0: r_matrix = r_set else: r_matrix = np.append(r_matrix, r_set, axis=0) #choose groups with best CC print "Selecting frames with best R" i_mean_r = np.argsort(np.mean(r_matrix, axis=1))[::-1] r_matrix_sorted = r_matrix[i_mean_r] frame_dup_files_sorted = np.array(frame_dup_files)[i_mean_r] frame_keys_sorted = np.array(frame_keys)[i_mean_r] frame_dup_files_sel = [] for frame_file, frame_key, r_set in zip(frame_dup_files_sorted, frame_keys_sorted, r_matrix_sorted): if frame_file not in frame_dup_files_sel: frame_dup_files_sel.append(frame_file) print frame_file, frame_key, np.mean(r_set) if len(frame_dup_files_sel) >= iparams.indexing_ambiguity.n_selected_frames: print 'Found all %6.0f good frames'%(len(frame_dup_files_sel)) break ## #rebuild observations and r_matrix frames = [(i, frame_dup_files_sel[i], iparams) for i in range(len(frame_dup_files_sel))] #get observations list print "Re-reading observations" alt_dict_results = pool_map( iterable=frames, func=get_obs_mproc, processes=iparams.n_processors) frame_dup_files = [] frame_keys = [] obs_list = [] for result in alt_dict_results: alt_dict, pickle_filename = result if alt_dict is not None: for key in alt_dict.keys(): frame_dup_files.append(pickle_filename) frame_keys.append(key) obs_list.append(alt_dict[key]) frames = [(i, frame_dup_files[i], frame_keys[i], obs_list[i], obs_list) for i in range(len(frame_dup_files))] #calculate r print "Re-calculating R" calc_r_results = pool_map( iterable=frames, func=calculate_r_mproc, processes=iparams.n_processors) frame_dup_files = [] frame_keys = [] r_matrix = [] for result in calc_r_results: if result is not None: pickle_filename, index_basis, r_set = result frame_dup_files.append(pickle_filename) frame_keys.append(index_basis) if len(r_matrix) == 0: r_matrix = r_set else: r_matrix = np.append(r_matrix, r_set, axis=0) print "Minimizing frame distance" idah = indamb_handler() x_set = idah.optimize(r_matrix, flag_plot=iparams.flag_plot) x_pickle = {'frame_dup_files':frame_dup_files, 'frame_keys':frame_keys, \ 'r_matrix':r_matrix, 'x_set':x_set} pickle.dump(x_pickle, open(iparams.run_no+'/index_ambiguity/x.out',"wb")) print "Clustering results" kmh = kmeans_handler() k = 2**(len(idah.get_observations(frame_dup_files[0], iparams))-1) centroids, labels = kmh.run(x_set, k, flag_plot=iparams.flag_plot) print "Get solution pickle" sample_fname = iparams.run_no+'/index_ambiguity/sample.lst' sol_pickle = idah.assign_basis(frame_dup_files, frame_keys, labels, k, sample_fname) pickle.dump(sol_pickle, open(sol_fname,"wb")) #if more frames found, merge the sample frames to get a reference set #that can be used for breaking the ambiguity. if n_frames > iparams.indexing_ambiguity.n_selected_frames: print "Breaking the indexing ambiguity for the remaining images." old_iparams_data = iparams.data[:] iparams.data = [sample_fname] iparams.indexing_ambiguity.index_basis_in = sol_fname grh = genref_handler() grh.run_by_params(iparams) mh = merge_handler() mh.run_by_params(iparams) DIR = iparams.run_no+'/mtz/' file_no_list = [int(fname.split('.')[0]) for fname in os.listdir(DIR)] if len(file_no_list) > 0: hklref_indamb = DIR + str(max(file_no_list)) + '.mtz' print "Bootstrap reference reflection set:", hklref_indamb #setup a list of remaining frames frame_files_remain = [] for frame in frame_files: if frame not in sol_pickle: frame_files_remain.append(frame) #determine index basis mxh = mx_handler() flag_ref_found, miller_array_ref = mxh.get_miller_array_from_reflection_file(hklref_indamb) frames = [(i, frame_files_remain[i], iparams, miller_array_ref) for i in range(len(frame_files_remain))] cc_results = pool_map( iterable=frames, func=solve_with_mtz_mproc, processes=iparams.n_processors) for result in cc_results: pickle_filename, index_basis = result sol_pickle[pickle_filename] = index_basis iparams.data = old_iparams_data[:] #write out solution pickle pickle.dump(sol_pickle, open(sol_fname,"wb")) #write out text output txt_out = "Solving indexing ambiguity complete. Solution file saved to "+sol_fname+"\n" f = open(iparams.run_no+'/log.txt', 'a') f.write(txt_out) f.close() return sol_fname, iparams
def get_twin_operators(self, obs): idah = indamb_handler() return idah.generate_twin_operators(obs_in)
def get_obs_mproc(args): frame_no, pickle_filename, iparams = args idah = indamb_handler() alt_dict = idah.get_observations(pickle_filename, iparams) return alt_dict, pickle_filename
def run(self, args): #read inputs from prime.postrefine.mod_input import process_input, read_pickles iparams, txt_out_input = process_input(args) print txt_out_input f = open(iparams.run_no+'/log.txt', 'w') f.write(txt_out_input) f.close() #if solution pickle is given, return the file name if iparams.indexing_ambiguity.index_basis_in is not None: if iparams.indexing_ambiguity.index_basis_in.endswith('.pickle'): return iparams.indexing_ambiguity.index_basis_in, iparams #read all integration pickles frame_files = read_pickles(iparams.data) n_frames = len(frame_files) if n_frames == 0: print "No integration pickle found. Exit program." return None, iparams #exit if no problem if self.should_terminate(iparams, frame_files[0]): print "No indexing ambiguity problem. Set index_ambiguity.mode = Forced and assigned_basis = list of basis formats to solve pseudo-twinning problem." return None, iparams #continue with (Auto - alt>1, find solution), (Auto - alt>1, mtz) #(Forced - assigned_basis, mtz), (Forced - assigned_basis, find solution) #************************************************* #if mtz file is given, use it to solve the problem sol_fname = iparams.run_no+'/index_ambiguity/solution_pickle.pickle' if iparams.indexing_ambiguity.index_basis_in is not None: if iparams.indexing_ambiguity.index_basis_in.endswith('.mtz'): mxh = mx_handler() flag_ref_found, miller_array_ref = mxh.get_miller_array_from_reflection_file(iparams.indexing_ambiguity.index_basis_in) if flag_ref_found == False: print "Reference mtz file not found. Set indexing_ambiguity.index_basis_in = None to enable auto generate the solutions." return None, iparams else: frames = [(i, frame_files[i], iparams, miller_array_ref) for i in range(n_frames)] cc_results = pool_map( iterable=frames, func=solve_with_mtz_mproc, processes=iparams.n_processors) sol_pickle = {} for result in cc_results: pickle_filename, index_basis = result sol_pickle[pickle_filename] = index_basis pickle.dump(sol_pickle, open(sol_fname,"wb")) return sol_fname, iparams #************************************************* #solve with Brehm & Diederichs - sample size n_sample_frames then bootstrap the rest frames = [(i, frame_files[i], iparams) for i in random.sample(range(n_frames), iparams.indexing_ambiguity.n_sample_frames)] #get observations list print "Reading observations" alt_dict_results = pool_map( iterable=frames, func=get_obs_mproc, processes=iparams.n_processors) frame_dup_files = [] frame_keys = [] obs_list = [] for result in alt_dict_results: alt_dict, pickle_filename = result if alt_dict is not None: for key in alt_dict.keys(): frame_dup_files.append(pickle_filename) frame_keys.append(key) obs_list.append(alt_dict[key]) frames = [(i, frame_dup_files[i], frame_keys[i], obs_list[i], obs_list) for i in range(len(frame_dup_files))] #calculate r print "Calculating R" calc_r_results = pool_map( iterable=frames, func=calculate_r_mproc, processes=iparams.n_processors) frame_dup_files = [] frame_keys = [] r_matrix = [] for result in calc_r_results: if result is not None: pickle_filename, index_basis, r_set = result frame_dup_files.append(pickle_filename) frame_keys.append(index_basis) if len(r_matrix) == 0: r_matrix = r_set else: r_matrix = np.append(r_matrix, r_set, axis=0) #choose groups with best CC print "Selecting frames with best R" i_mean_r = np.argsort(np.mean(r_matrix, axis=1))[::-1] r_matrix_sorted = r_matrix[i_mean_r] frame_dup_files_sorted = np.array(frame_dup_files)[i_mean_r] frame_keys_sorted = np.array(frame_keys)[i_mean_r] frame_dup_files_sel = [] for frame_file, frame_key, r_set in zip(frame_dup_files_sorted, frame_keys_sorted, r_matrix_sorted): if frame_file not in frame_dup_files_sel: frame_dup_files_sel.append(frame_file) print frame_file, frame_key, np.mean(r_set) if len(frame_dup_files_sel) >= iparams.indexing_ambiguity.n_selected_frames: print 'Found all %6.0f good frames'%(len(frame_dup_files_sel)) break ## #rebuild observations and r_matrix frames = [(i, frame_dup_files_sel[i], iparams) for i in range(len(frame_dup_files_sel))] #get observations list print "Re-reading observations" alt_dict_results = pool_map( iterable=frames, func=get_obs_mproc, processes=iparams.n_processors) frame_dup_files = [] frame_keys = [] obs_list = [] for result in alt_dict_results: alt_dict, pickle_filename = result if alt_dict is not None: for key in alt_dict.keys(): frame_dup_files.append(pickle_filename) frame_keys.append(key) obs_list.append(alt_dict[key]) frames = [(i, frame_dup_files[i], frame_keys[i], obs_list[i], obs_list) for i in range(len(frame_dup_files))] #calculate r print "Re-calculating R" calc_r_results = pool_map( iterable=frames, func=calculate_r_mproc, processes=iparams.n_processors) frame_dup_files = [] frame_keys = [] r_matrix = [] for result in calc_r_results: if result is not None: pickle_filename, index_basis, r_set = result frame_dup_files.append(pickle_filename) frame_keys.append(index_basis) if len(r_matrix) == 0: r_matrix = r_set else: r_matrix = np.append(r_matrix, r_set, axis=0) print "Minimizing frame distance" idah = indamb_handler() x_set = idah.optimize(r_matrix, flag_plot=iparams.flag_plot) x_pickle = {'frame_dup_files':frame_dup_files, 'frame_keys':frame_keys, \ 'r_matrix':r_matrix, 'x_set':x_set} pickle.dump(x_pickle, open(iparams.run_no+'/index_ambiguity/x.out',"wb")) print "Clustering results" kmh = kmeans_handler() k = 2**(len(idah.get_observations(frame_dup_files[0], iparams))-1) centroids, labels = kmh.run(x_set, k, flag_plot=iparams.flag_plot) print "Get solution pickle" sample_fname = iparams.run_no+'/index_ambiguity/sample.lst' sol_pickle = idah.assign_basis(frame_dup_files, frame_keys, labels, k, sample_fname) pickle.dump(sol_pickle, open(sol_fname,"wb")) #if more frames found, merge the sample frames to get a reference set #that can be used for breaking the ambiguity. if n_frames > iparams.indexing_ambiguity.n_selected_frames: print "Breaking the indexing ambiguity for the remaining images." old_iparams_data = iparams.data[:] iparams.data = [sample_fname] iparams.indexing_ambiguity.index_basis_in = sol_fname grh = genref_handler() grh.run_by_params(iparams) mh = merge_handler() mh.run_by_params(iparams) DIR = iparams.run_no+'/mtz/' file_no_list = [int(fname.split('.')[0]) for fname in os.listdir(DIR)] if len(file_no_list) > 0: hklref_indamb = DIR + str(max(file_no_list)) + '.mtz' print "Bootstrap reference reflection set:", hklref_indamb #setup a list of remaining frames frame_files_remain = [] for frame in frame_files: if frame not in sol_pickle: frame_files_remain.append(frame) #determine index basis mxh = mx_handler() flag_ref_found, miller_array_ref = mxh.get_miller_array_from_reflection_file(hklref_indamb) frames = [(i, frame_files_remain[i], iparams, miller_array_ref) for i in range(len(frame_files_remain))] cc_results = pool_map( iterable=frames, func=solve_with_mtz_mproc, processes=iparams.n_processors) for result in cc_results: pickle_filename, index_basis = result sol_pickle[pickle_filename] = index_basis iparams.data = old_iparams_data[:] #write out solution pickle pickle.dump(sol_pickle, open(sol_fname,"wb")) #write out text output txt_out = "Solving indexing ambiguity complete. Solution file saved to "+sol_fname+"\n" f = open(iparams.run_no+'/log.txt', 'a') f.write(txt_out) f.close() return sol_fname, iparams
def get_twin_operators(self, obs): idah = indamb_handler() return idah.generate_twin_operators(obs_in)
def get_obs_mproc(args): frame_no, pickle_filename, iparams = args idah = indamb_handler() alt_dict = idah.get_observations(pickle_filename, iparams) return alt_dict, pickle_filename
def calculate_r_mproc(args): frame_no, pickle_filename, index_basis, obs_main, obs_list = args idah = indamb_handler() r_set, txt_out = idah.calc_r(frame_no, pickle_filename, index_basis, obs_main, obs_list) print txt_out return pickle_filename, index_basis, r_set
def solve_with_mtz_mproc(args): frame_no, pickle_filename, iparams, miller_array_ref = args idah = indamb_handler() index_basis, txt_out = idah.calc_cc(frame_no, pickle_filename, iparams, miller_array_ref) print txt_out return pickle_filename, index_basis
- 1] if cc_bin_low < cc_bin_low_thres: plt.subplot(2, 1, 1) plt.plot(oodsqrt_bins, cc_bins) plt.title('CC by resolutions for cc < %6.2f' % (cc_bin_low_thres)) plt.xlabel('1/d^2') plt.ylim([0, 1]) plt.grid(True) else: plt.subplot(2, 1, 2) plt.plot(oodsqrt_bins, cc_bins) plt.title('CC by resolutions for cc > %6.2f' % (cc_bin_low_thres)) plt.xlabel('1/d^2') plt.ylim([0, 1]) plt.grid(True) cn_i += 1 plt.show() #print twin operators idah = indamb_handler() for pickle_filename in frame_files: pickle_filename_arr = pickle_filename.split('/') pickle_filename_only = pickle_filename_arr[len(pickle_filename_arr) - 1] observations_pickle = pickle.load(open(pickle_filename, "rb")) observations = observations_pickle["observations"][0] operators = idah.generate_twin_operators(observations, flag_all=True) ops_hkl = [op.operator.r().as_hkl() for op in operators] print pickle_filename_only, ops_hkl
cn_i = 0 for cc_bins, d_bins, oodsqrt_bins, cc_bin_low, pickle_filename in zip(cc_bins_set, d_bins_set, oodsqr_bins_set, cc_bin_low_set, frame_files): pickle_filename_arr = pickle_filename.split('/') pickle_filename_only = pickle_filename_arr[len(pickle_filename_arr)-1] if cc_bin_low < cc_bin_low_thres: plt.subplot(2,1,1) plt.plot(oodsqrt_bins, cc_bins) plt.title('CC by resolutions for cc < %6.2f'%(cc_bin_low_thres)) plt.xlabel('1/d^2') plt.ylim([0,1]) plt.grid(True) else: plt.subplot(2,1,2) plt.plot(oodsqrt_bins, cc_bins) plt.title('CC by resolutions for cc > %6.2f'%(cc_bin_low_thres)) plt.xlabel('1/d^2') plt.ylim([0,1]) plt.grid(True) cn_i += 1 plt.show() #print twin operators idah = indamb_handler() for pickle_filename in frame_files: pickle_filename_arr = pickle_filename.split('/') pickle_filename_only = pickle_filename_arr[len(pickle_filename_arr)-1] observations_pickle = pickle.load(open(pickle_filename,"rb")) observations = observations_pickle["observations"][0] operators = idah.generate_twin_operators(observations, flag_all=True) ops_hkl = [op.operator.r().as_hkl() for op in operators] print pickle_filename_only, ops_hkl