def align_all_pairs(avgs, dj, img_db, n_chunk=1000, redis_host=None): """ because python variables are references, it is fine to prepare large amount of tasks, whose prameters points to a small numbers of images """ # print 'align_all_pairs' ts = {} for d in dj: v = img_db[d['subtomogram']] m = img_db[d['mask']] for k in avgs: t = dict() t['uuid'] = str(uuid.uuid4()) # t['module'] = 'tomominer.align.util' t['module'] = 'aitom.align.fast.util' t['method'] = 'align_vols' t['subtomogram_id'] = d['subtomogram'] t['template_id'] = k a_t = dict() a_t['v1'] = avgs[k]['v'] a_t['m1'] = avgs[k]['m'] a_t['v2'] = v a_t['m2'] = m a_t['L'] = 36 t['kwargs'] = a_t ts[t['uuid']] = t # al[i][k] = TAU.align_vols(v1=avgs[k]['v'], m1=avgs[k]['m'], v2=v, m2=m, L=36) # al[i][k]['template_id'] = k from collections import defaultdict al = defaultdict(dict) import aitom.parallel.multiprocessing.util as PARALLEL if redis_host is None: tr_s = [_ for _ in PARALLEL.run_iterator(ts)] else: tr_s = [ _ for _ in PARALLEL.run_iterator( ts, n_chunk=n_chunk, redis_host=redis_host) ] for tr in tr_s: i = tr['id'] r = tr['result'] al[ts[i]['subtomogram_id']][ts[i]['template_id']] = r al[ts[i]['subtomogram_id']][ ts[i]['template_id']]['template_id'] = ts[i]['template_id'] return al
def align_all_pairs(avgs, dj, img_db, n_chunk=1000, redis_host=None): # print 'align_all_pairs' ts = {} for d in dj: v = img_db[d['subtomogram']] m = img_db[d['mask']] for k in avgs: t = {} t['uuid'] = str(uuid.uuid4()) # t['module'] = 'tomominer.align.util' t['module'] = 'aitom.align.fast.util' t['method'] = 'align_vols' t['subtomogram_id'] = d['subtomogram'] t['template_id'] = k a_t = {} a_t['v1'] = avgs[k]['v'] a_t['m1'] = avgs[k]['m'] a_t['v2'] = v a_t['m2'] = m a_t['L'] = 36 t['kwargs'] = a_t ts[t['uuid']] = t # al[i][k] = TAU.align_vols(v1=avgs[k]['v'], m1=avgs[k]['m'], v2=v, m2=m, L=36) # al[i][k]['template_id'] = k from collections import defaultdict al = defaultdict(dict) if redis_host is None: # import tomominer.parallel.multiprocessing.util as PARALLEL import aitom.parallel.multiprocessing.util as PARALLEL tr_s = [_ for _ in PARALLEL.run_iterator(ts)] else: # import tomominer.parallel.redis.job_queue.batch_run as PARALLEL import aitom.parallel.multiprocessing.util as PARALLEL tr_s = [ _ for _ in PARALLEL.run_iterator( ts, n_chunk=n_chunk, redis_host=redis_host) ] for tr in tr_s: i = tr['id'] r = tr['result'] al[ts[i]['subtomogram_id']][ts[i]['template_id']] = r al[ts[i]['subtomogram_id']][ ts[i]['template_id']]['template_id'] = ts[i]['template_id'] return al
def compute_trans_list(theta, img_data, use_voronoi): N = theta['N'] K = theta['K'] n = theta['n'] theta['trans_list'] = [[None for _ in range(K)] for _ in range(N)] # Compute the list of optimal transforms against all A_k's # for all i's in parallel tasks = {} for k_ in range(K): A_k = theta['A'][k_] for j,d in enumerate(img_data['dj']): t = {} t['uuid'] = str(uuid.uuid4()) t['module'] = 'aitom.average.ml.faml.faml' t['method'] = 'model_based_align_help' t['kwargs'] = {'img_db_path':img_data['db_path'], 'd':d, 'A_k':A_k, 'n':n, 'i':j, 'k':k_} tasks[t['uuid']] = t if len(sys.argv) > 1: rt_s = [_ for _ in TPRJB.run_iterator(tasks, redis_host = sys.argv[1], redis_port=6379, redis_password='******')] else: rt_s = [_ for _ in TPMU.run_iterator(tasks, worker_num = MULTIPROCESSING_WORKER_NUM)] for rt in rt_s: j = rt['result']['i'] k_ = rt['result']['k'] result = rt['result']['transforms'] theta['trans_list'][j][k_] = result if use_voronoi: # Compute the weights for each configuration compute_voronoi_weights(theta)
def update_xi(img_data, theta, use_voronoi): compute_prob(img_data=img_data, theta=theta) K = theta['K'] J = theta['J'] N = theta['N'] xi_sq_new = 0 tasks = {} for i in range(N): t = {} t['uuid'] = str(uuid.uuid4()) t['module'] = 'aitom.average.ml.faml.faml' t['method'] = 'update_xi_help' t['kwargs'] = {'i': i, 'theta':theta, 'use_voronoi':use_voronoi} tasks[t['uuid']] = t if len(sys.argv) > 1: rt_s = [_ for _ in TPRJB.run_iterator(tasks, redis_host = sys.argv[1], redis_port=6379, redis_password='******')] else: rt_s = [_ for _ in TPMU.run_iterator(tasks, worker_num = MULTIPROCESSING_WORKER_NUM)] for rt in rt_s: xi_sq_new += rt['result']['result'] return max(eps, (xi_sq_new / (3 * N)) ** 0.5)
def update_alpha(img_data, theta, use_voronoi): compute_prob(img_data=img_data, theta=theta) K = theta['K'] N = theta['N'] probs = np.zeros([N, K]) tasks = {} for k in range(K): for i,d in enumerate(img_data['dj']): t = {} t['uuid'] = str(uuid.uuid4()) t['module'] = 'aitom.average.ml.faml.faml' t['method'] = 'update_alpha_help' t['kwargs'] = {'i': i, 'theta':theta, 'k':k, 'use_voronoi':use_voronoi} tasks[t['uuid']] = t if len(sys.argv) > 1: rt_s = [_ for _ in TPRJB.run_iterator(tasks, redis_host = sys.argv[1], redis_port=6379, redis_password='******')] else: rt_s = [_ for _ in TPMU.run_iterator(tasks, worker_num = MULTIPROCESSING_WORKER_NUM)] for rt in rt_s: k = rt['result']['k'] i = rt['result']['i'] probs[i][k] = rt['result']['result'] alpha_new = np.sum(probs, axis=0) / N theta['predictions'] = np.argmax(probs, axis=1) return alpha_new
def compute_prob(img_data, theta): dj = img_data['dj'] try: assert theta['N'] == len(dj) except: raise Exception("Error in compute_prob: Inconsistent data dimensions!") N = theta['N'] K = theta['K'] n = theta['n'] theta['ln_prob'] = [[None for _ in range(K)] for _ in range(N)] tasks = {} for k in range(K): A_k = theta['A'][k] for i,d in enumerate(img_data['dj']): trans_list = theta['trans_list'][i][k] t = {} t['uuid'] = str(uuid.uuid4()) t['module'] = 'aitom.average.ml.faml.faml' t['method'] = 'model_based_prob_help' t['kwargs'] = {'img_db_path':img_data['db_path'], 'd':d, 'A_k':A_k, 'n':n, 'i':i, 'k':k, 'J': theta['J'], 'trans_list':trans_list, 'sigma_sq': theta['sigma_sq'], 'xi': theta['xi'], 'alpha': theta['alpha']} tasks[t['uuid']] = t if len(sys.argv) > 1: rt_s = [_ for _ in TPRJB.run_iterator(tasks, redis_host = sys.argv[1], redis_port=6379, redis_password='******')] else: rt_s = [_ for _ in TPMU.run_iterator(tasks, worker_num = MULTIPROCESSING_WORKER_NUM)] for rt in rt_s: i = rt['result']['i'] k = rt['result']['k'] ln_prob_list = rt['result']['ln_prob'] theta['ln_prob'][i][k] = ln_prob_list
def compute_voronoi_weights(theta): N = theta['N'] K = theta['K'] n = theta['n'] theta['voronoi'] = [[None for _ in range(K)] for _ in range(N)] tasks = {} for k in range(K): for i in range(N): trans_list = theta['trans_list'][i][k] t = {} t['uuid'] = str(uuid.uuid4()) t['module'] = 'aitom.tomominer.hypervolume.utils' t['method'] = 'voronoi_weights_6d' t['kwargs'] = {'phis':trans_list} t['i'] = i t['k'] = k tasks[t['uuid']] = t del i, k if len(sys.argv) > 1: rt_s = [_ for _ in TPRJB.run_iterator(tasks, redis_host = sys.argv[1], redis_port=6379, redis_password='******')] else: rt_s = [_ for _ in TPMU.run_iterator(tasks, worker_num = MULTIPROCESSING_WORKER_NUM)] for rt in rt_s: i = tasks[rt['id']]['i'] k = tasks[rt['id']]['k'] theta['voronoi'][i][k] = rt['result']
def update_sigma(img_data, theta, use_voronoi, reg=True): """ Updates the sigma_sq values in the parameters dictionary theta """ compute_prob(img_data=img_data, theta=theta) K = theta['K'] J = theta['J'] N = theta['N'] A = theta['A'] n = theta['n'] sigma_sq_old = theta['sigma_sq'] # Regularization term reg_term = 0 if reg and theta['theta_reg'] > 0: for k in range(K): for l in range(K): reg_term += np.sum(np.square(np.absolute(A[k] - A[l]))) reg_term *= theta['theta_reg'] tasks = {} for i, d in enumerate(img_data['dj']): t = dict() t['uuid'] = str(uuid.uuid4()) t['module'] = 'aitom.average.ml.faml.faml' t['method'] = 'update_sigma_help' t['kwargs'] = { 'img_db_path': img_data['db_path'], 'i': i, 'theta': theta, 'd': d, 'use_voronoi': use_voronoi } tasks[t['uuid']] = t if len(sys.argv) > 1: rt_s = [ _ for _ in TPRJB.run_iterator( tasks, redis_host=sys.argv[1], redis_port=6379, redis_password= '******') ] else: rt_s = [ _ for _ in TPMU.run_iterator(tasks, worker_num=MULTIPROCESSING_WORKER_NUM) ] sigma_sq_new = 0 for rt in rt_s: sigma_sq_new += rt['result']['result'] if reg: sigma_sq_new += reg_term return sigma_sq_new / (N * J)
def update_a(img_data, theta, alpha, use_voronoi, reg=True): dj = img_data['dj'] compute_prob(img_data=img_data, theta=theta) K = theta['K'] J = theta['J'] N = theta['N'] A_old = theta['A'] n = theta['n'] A_new = np.zeros([K, n, n, n], dtype=np.complex128) # Regularization terms reg_num = theta['theta_reg'] * np.sum(A_old, axis=0) reg_denom = theta['theta_reg'] * K tasks = {} for k in range(K): for i, d in enumerate(img_data['dj']): t = {} t['uuid'] = str(uuid.uuid4()) t['module'] = 'aitom.average.ml.faml.faml' t['method'] = 'update_a_help' t['kwargs'] = { 'img_db_path': img_data['db_path'], 'd': d, 'i': i, 'theta': theta, 'k': k, 'use_voronoi': use_voronoi } tasks[t['uuid']] = t if len(sys.argv) > 1: rt_s = [ _ for _ in TPRJB.run_iterator( tasks, redis_host=sys.argv[1], redis_port=6379, redis_password= '******') ] else: rt_s = [ _ for _ in TPMU.run_iterator(tasks, worker_num=MULTIPROCESSING_WORKER_NUM) ] for rt in rt_s: k = rt['result']['k'] A_new[k] += rt['result']['result'] for k in range(K): if reg and theta['theta_reg'] > 0: A_new[k] += reg_num A_new[k] /= alpha[k] * N + reg_denom else: A_new[k] /= alpha[k] * N return A_new
def compute_voronoi_weights(theta): """ Compute the weights of each configuration by approximating the hypervolume of the voronoi region of each configuration in the 6-dimensional space of all configurations """ N = theta['N'] K = theta['K'] n = theta['n'] theta['voronoi'] = [[None for _ in range(K)] for _ in range(N)] tasks = {} for k in range(K): for i in range(N): trans_list = theta['trans_list'][i][k] t = dict() t['uuid'] = str(uuid.uuid4()) t['module'] = 'aitom.geometry.volume.hypervolume.utils' t['method'] = 'voronoi_weights_6d' t['kwargs'] = {'phis': trans_list} t['i'] = i t['k'] = k tasks[t['uuid']] = t del i, k if len(sys.argv) > 1: rt_s = [ _ for _ in TPRJB.run_iterator( tasks, redis_host=sys.argv[1], redis_port=6379, redis_password= '******') ] else: rt_s = [ _ for _ in TPMU.run_iterator(tasks, worker_num=MULTIPROCESSING_WORKER_NUM) ] for rt in rt_s: i = tasks[rt['id']]['i'] k = tasks[rt['id']]['k'] theta['voronoi'][i][k] = rt['result']