def calculate_r_mproc(args):
frame_no, pickle_filename, obs_main, obs_list = args
isoch = isoform_cluster_handler()
r_set, main_obs, txt_out = isoch.calc_r(frame_no, pickle_filename,
obs_main, obs_list)
print txt_out
return pickle_filename, r_set, main_obs
def solve_with_mtz_mproc(args):
frame_no, pickle_filename, main_obs, iparams, miller_array_ref_set = args
isoch = isoform_cluster_handler()
cluster_id, txt_out = isoch.calc_cc(frame_no, pickle_filename, main_obs,
iparams, miller_array_ref_set)
print txt_out
return pickle_filename, cluster_id
def run(self, args):
#read inputs
from prime.postrefine.mod_input import process_input, read_pickles
iparams, txt_out_input = process_input(args)
print txt_out_input
with open(os.path.join(iparams.run_no, self.module_name, 'log.txt'),
'w') as f:
f.write(txt_out_input)
#read all integration pickles
frame_files = read_pickles(iparams.data)
n_frames = len(frame_files)
if n_frames == 0:
print "No integration pickle found. Exit program."
return None, iparams
#start
if iparams.isoform_cluster.isorefin:
#get collection of iso. ref. reflection set.
mxh = mx_handler()
miller_array_ref_set = []
for isorefin in iparams.isoform_cluster.isorefin:
flag_ref_found, miller_array_ref = mxh.get_miller_array_from_reflection_file(
isorefin)
if flag_ref_found:
miller_array_ref_set.append(miller_array_ref)
#get observation list
frame_files_sel, obs_list = self.get_observation_set(
iparams, frame_files, n_frames)
if miller_array_ref_set:
frames = [(i, frame_files_sel[i], obs_list[i], iparams,
miller_array_ref_set) for i in range(len(obs_list))]
cc_results = pool_map(iterable=frames,
func=solve_with_mtz_mproc,
processes=iparams.n_processors)
sol_pickle = {}
for result in cc_results:
pickle_filename, cluster_id = result
sol_pickle[pickle_filename] = cluster_id
write_out_solutions(iparams, sol_pickle)
txt_out = "Cluster images with given " + str(
len(miller_array_ref_set)
) + " mtz files completed. Use cluster_0.lst - cluster_k.lst (for k clusters) for merging.\n"
print txt_out
with open(
os.path.join(iparams.run_no, self.module_name,
'log.txt'), 'a') as f:
f.write(txt_out)
return
#*************************************************
#solve with Brehm & Diederichs - sample size n_sample_frames then bootstrap the rest
txt_out = "Cluster images with B&D algorithms.\n"
frame_files_sel, obs_list = self.get_observation_set(
iparams, frame_files, iparams.isoform_cluster.n_sample_frames)
frames = [(i, frame_files_sel[i], obs_list[i], obs_list)
for i in range(len(frame_files_sel))]
#calculate r
print "Calculating R"
calc_r_results = pool_map(iterable=frames,
func=calculate_r_mproc,
processes=iparams.n_processors)
frame_files_sel = []
r_matrix = []
obs_list = []
for result in calc_r_results:
if result:
pickle_filename, r_set, obs = result
frame_files_sel.append(pickle_filename)
obs_list.append(obs)
if len(r_matrix) == 0:
r_matrix = r_set
else:
r_matrix = np.append(r_matrix, r_set, axis=0)
#choose groups with best R
print "Selecting frames with best R"
i_mean_r = np.argsort(np.mean(r_matrix, axis=1))[::-1]
r_matrix_sorted = r_matrix[i_mean_r]
frame_files_sorted = np.array(frame_files_sel)[i_mean_r]
obs_list_sorted = np.array(obs_list)[i_mean_r]
frame_files_sel = []
obs_sel = []
for frame_file, r_set, obs in zip(frame_files_sorted, r_matrix_sorted,
obs_list_sorted):
if frame_file not in frame_files_sel:
frame_files_sel.append(frame_file)
obs_sel.append(obs)
print frame_file, np.mean(r_set)
if len(frame_files_sel
) >= iparams.isoform_cluster.n_selected_frames:
print 'Found all %6.0f good frames' % (
len(frame_files_sel))
break
#Recalculate r for the new selected list
frames = [(i, frame_files_sel[i], obs_sel[i], obs_sel)
for i in range(len(frame_files_sel))]
print "Re-calculating R"
calc_r_results = pool_map(iterable=frames,
func=calculate_r_mproc,
processes=iparams.n_processors)
frame_files_sel = []
r_matrix = []
obs_list = []
for result in calc_r_results:
if result:
pickle_filename, r_set, obs = result
frame_files_sel.append(pickle_filename)
obs_list.append(obs)
if len(r_matrix) == 0:
r_matrix = r_set
else:
r_matrix = np.append(r_matrix, r_set, axis=0)
print "Minimizing frame distance"
isoch = isoform_cluster_handler()
x_set = isoch.optimize(r_matrix, flag_plot=iparams.flag_plot)
print "Clustering results"
kmh = kmeans_handler()
k = iparams.isoform_cluster.n_clusters
centroids, labels = kmh.run(x_set, k, flag_plot=iparams.flag_plot)
print "Get solution pickle and cluster files list"
sol_pickle, cluster_files = isoch.assign_cluster(frame_files_sel, labels, k, \
os.path.join(iparams.run_no,self.module_name))
#if more frames found, merge the sample frames to get a reference set
#that can be used for breaking the ambiguity.
if n_frames > iparams.isoform_cluster.n_selected_frames:
print "Assign cluster_id for the remaining images."
old_iparams_data = iparams.data[:]
miller_array_ref_set = []
from prime.command_line.postrefine import scale_frames, merge_frames
for i in range(k):
#generate a reference set from solved frames
with open(cluster_files[i]) as f:
frame_files_processed = f.read().split('\n')[:-1]
scaled_pres_set = scale_frames(
range(len(frame_files_processed)), frame_files_processed,
iparams)
mdh, txt_merge_out = merge_frames(scaled_pres_set, iparams, \
mtz_out_prefix=os.path.join(self.module_name,'cluster_'+str(i)))
miller_array_ref_set.append(mdh.miller_array_merge)
txt_out += txt_merge_out
#setup a list of remaining frames
frame_files_remain = [
frame for frame in frame_files if frame not in sol_pickle
]
frame_files_remain_sel, obs_remain_sel_list = self.get_observation_set(iparams, \
frame_files_remain, len(frame_files_remain))
frames = [(i, frame_files_remain_sel[i], obs_remain_sel_list[i],
iparams, miller_array_ref_set)
for i in range(len(obs_remain_sel_list))]
cc_results = pool_map(iterable=frames,
func=solve_with_mtz_mproc,
processes=iparams.n_processors)
for result in cc_results:
if result:
pickle_filename, cluster_id = result
sol_pickle[pickle_filename] = cluster_id
iparams.data = old_iparams_data[:]
#write out solution pickle
write_out_solutions(iparams, sol_pickle)
#write out text output
txt = "Cluster images completed. Use cluster_0.lst - cluster_k.lst (for k clusters) for merging.\n"
txt_out += txt
print txt
with open(os.path.join(iparams.run_no, self.module_name, 'log.txt'),
'a') as f:
f.write(txt_out)
def get_obs_mproc(args):
frame_no, pickle_filename, iparams = args
isoch = isoform_cluster_handler()
obs_asu = isoch.get_observations(pickle_filename, iparams)
return obs_asu, pickle_filename
