def align_all_pairs(avgs, dj, img_db, n_chunk=1000, redis_host=None):
"""
because python variables are references, it is fine to prepare large amount of tasks, whose prameters points to a small numbers of images
"""
# print 'align_all_pairs'
ts = {}
for d in dj:
v = img_db[d['subtomogram']]
m = img_db[d['mask']]
for k in avgs:
t = dict()
t['uuid'] = str(uuid.uuid4())
# t['module'] = 'tomominer.align.util'
t['module'] = 'aitom.align.fast.util'
t['method'] = 'align_vols'
t['subtomogram_id'] = d['subtomogram']
t['template_id'] = k
a_t = dict()
a_t['v1'] = avgs[k]['v']
a_t['m1'] = avgs[k]['m']
a_t['v2'] = v
a_t['m2'] = m
a_t['L'] = 36
t['kwargs'] = a_t
ts[t['uuid']] = t
# al[i][k] = TAU.align_vols(v1=avgs[k]['v'], m1=avgs[k]['m'], v2=v, m2=m, L=36)
# al[i][k]['template_id'] = k
from collections import defaultdict
al = defaultdict(dict)
import aitom.parallel.multiprocessing.util as PARALLEL
if redis_host is None:
tr_s = [_ for _ in PARALLEL.run_iterator(ts)]
else:
tr_s = [
_ for _ in PARALLEL.run_iterator(
ts, n_chunk=n_chunk, redis_host=redis_host)
]
for tr in tr_s:
i = tr['id']
r = tr['result']
al[ts[i]['subtomogram_id']][ts[i]['template_id']] = r
al[ts[i]['subtomogram_id']][
ts[i]['template_id']]['template_id'] = ts[i]['template_id']
return al
def align_all_pairs(avgs, dj, img_db, n_chunk=1000, redis_host=None):
# print 'align_all_pairs'
ts = {}
for d in dj:
v = img_db[d['subtomogram']]
m = img_db[d['mask']]
for k in avgs:
t = {}
t['uuid'] = str(uuid.uuid4())
# t['module'] = 'tomominer.align.util'
t['module'] = 'aitom.align.fast.util'
t['method'] = 'align_vols'
t['subtomogram_id'] = d['subtomogram']
t['template_id'] = k
a_t = {}
a_t['v1'] = avgs[k]['v']
a_t['m1'] = avgs[k]['m']
a_t['v2'] = v
a_t['m2'] = m
a_t['L'] = 36
t['kwargs'] = a_t
ts[t['uuid']] = t
# al[i][k] = TAU.align_vols(v1=avgs[k]['v'], m1=avgs[k]['m'], v2=v, m2=m, L=36)
# al[i][k]['template_id'] = k
from collections import defaultdict
al = defaultdict(dict)
if redis_host is None:
# import tomominer.parallel.multiprocessing.util as PARALLEL
import aitom.parallel.multiprocessing.util as PARALLEL
tr_s = [_ for _ in PARALLEL.run_iterator(ts)]
else:
# import tomominer.parallel.redis.job_queue.batch_run as PARALLEL
import aitom.parallel.multiprocessing.util as PARALLEL
tr_s = [
_ for _ in PARALLEL.run_iterator(
ts, n_chunk=n_chunk, redis_host=redis_host)
]
for tr in tr_s:
i = tr['id']
r = tr['result']
al[ts[i]['subtomogram_id']][ts[i]['template_id']] = r
al[ts[i]['subtomogram_id']][
ts[i]['template_id']]['template_id'] = ts[i]['template_id']
return al
def compute_trans_list(theta, img_data, use_voronoi):
N = theta['N']
K = theta['K']
n = theta['n']
theta['trans_list'] = [[None for _ in range(K)] for _ in range(N)]
# Compute the list of optimal transforms against all A_k's
# for all i's in parallel
tasks = {}
for k_ in range(K):
A_k = theta['A'][k_]
for j,d in enumerate(img_data['dj']):
t = {}
t['uuid'] = str(uuid.uuid4())
t['module'] = 'aitom.average.ml.faml.faml'
t['method'] = 'model_based_align_help'
t['kwargs'] = {'img_db_path':img_data['db_path'], 'd':d, 'A_k':A_k, 'n':n, 'i':j, 'k':k_}
tasks[t['uuid']] = t
if len(sys.argv) > 1:
rt_s = [_ for _ in TPRJB.run_iterator(tasks, redis_host = sys.argv[1], redis_port=6379, redis_password='******')]
else:
rt_s = [_ for _ in TPMU.run_iterator(tasks, worker_num = MULTIPROCESSING_WORKER_NUM)]
for rt in rt_s:
j = rt['result']['i']
k_ = rt['result']['k']
result = rt['result']['transforms']
theta['trans_list'][j][k_] = result
if use_voronoi:
# Compute the weights for each configuration
compute_voronoi_weights(theta)
def update_xi(img_data, theta, use_voronoi):
compute_prob(img_data=img_data, theta=theta)
K = theta['K']
J = theta['J']
N = theta['N']
xi_sq_new = 0
tasks = {}
for i in range(N):
t = {}
t['uuid'] = str(uuid.uuid4())
t['module'] = 'aitom.average.ml.faml.faml'
t['method'] = 'update_xi_help'
t['kwargs'] = {'i': i, 'theta':theta, 'use_voronoi':use_voronoi}
tasks[t['uuid']] = t
if len(sys.argv) > 1:
rt_s = [_ for _ in TPRJB.run_iterator(tasks, redis_host = sys.argv[1], redis_port=6379, redis_password='******')]
else:
rt_s = [_ for _ in TPMU.run_iterator(tasks, worker_num = MULTIPROCESSING_WORKER_NUM)]
for rt in rt_s:
xi_sq_new += rt['result']['result']
return max(eps, (xi_sq_new / (3 * N)) ** 0.5)
def update_alpha(img_data, theta, use_voronoi):
compute_prob(img_data=img_data, theta=theta)
K = theta['K']
N = theta['N']
probs = np.zeros([N, K])
tasks = {}
for k in range(K):
for i,d in enumerate(img_data['dj']):
t = {}
t['uuid'] = str(uuid.uuid4())
t['module'] = 'aitom.average.ml.faml.faml'
t['method'] = 'update_alpha_help'
t['kwargs'] = {'i': i, 'theta':theta, 'k':k, 'use_voronoi':use_voronoi}
tasks[t['uuid']] = t
if len(sys.argv) > 1:
rt_s = [_ for _ in TPRJB.run_iterator(tasks, redis_host = sys.argv[1], redis_port=6379, redis_password='******')]
else:
rt_s = [_ for _ in TPMU.run_iterator(tasks, worker_num = MULTIPROCESSING_WORKER_NUM)]
for rt in rt_s:
k = rt['result']['k']
i = rt['result']['i']
probs[i][k] = rt['result']['result']
alpha_new = np.sum(probs, axis=0) / N
theta['predictions'] = np.argmax(probs, axis=1)
return alpha_new
def compute_prob(img_data, theta):
dj = img_data['dj']
try:
assert theta['N'] == len(dj)
except:
raise Exception("Error in compute_prob: Inconsistent data dimensions!")
N = theta['N']
K = theta['K']
n = theta['n']
theta['ln_prob'] = [[None for _ in range(K)] for _ in range(N)]
tasks = {}
for k in range(K):
A_k = theta['A'][k]
for i,d in enumerate(img_data['dj']):
trans_list = theta['trans_list'][i][k]
t = {}
t['uuid'] = str(uuid.uuid4())
t['module'] = 'aitom.average.ml.faml.faml'
t['method'] = 'model_based_prob_help'
t['kwargs'] = {'img_db_path':img_data['db_path'], 'd':d, 'A_k':A_k, 'n':n, 'i':i, 'k':k, 'J': theta['J'], 'trans_list':trans_list, 'sigma_sq': theta['sigma_sq'], 'xi': theta['xi'], 'alpha': theta['alpha']}
tasks[t['uuid']] = t
if len(sys.argv) > 1:
rt_s = [_ for _ in TPRJB.run_iterator(tasks, redis_host = sys.argv[1], redis_port=6379, redis_password='******')]
else:
rt_s = [_ for _ in TPMU.run_iterator(tasks, worker_num = MULTIPROCESSING_WORKER_NUM)]
for rt in rt_s:
i = rt['result']['i']
k = rt['result']['k']
ln_prob_list = rt['result']['ln_prob']
theta['ln_prob'][i][k] = ln_prob_list
def compute_voronoi_weights(theta):
N = theta['N']
K = theta['K']
n = theta['n']
theta['voronoi'] = [[None for _ in range(K)] for _ in range(N)]
tasks = {}
for k in range(K):
for i in range(N):
trans_list = theta['trans_list'][i][k]
t = {}
t['uuid'] = str(uuid.uuid4())
t['module'] = 'aitom.tomominer.hypervolume.utils'
t['method'] = 'voronoi_weights_6d'
t['kwargs'] = {'phis':trans_list}
t['i'] = i
t['k'] = k
tasks[t['uuid']] = t
del i, k
if len(sys.argv) > 1:
rt_s = [_ for _ in TPRJB.run_iterator(tasks, redis_host = sys.argv[1], redis_port=6379, redis_password='******')]
else:
rt_s = [_ for _ in TPMU.run_iterator(tasks, worker_num = MULTIPROCESSING_WORKER_NUM)]
for rt in rt_s:
i = tasks[rt['id']]['i']
k = tasks[rt['id']]['k']
theta['voronoi'][i][k] = rt['result']
def update_sigma(img_data, theta, use_voronoi, reg=True):
"""
Updates the sigma_sq values in the parameters dictionary theta
"""
compute_prob(img_data=img_data, theta=theta)
K = theta['K']
J = theta['J']
N = theta['N']
A = theta['A']
n = theta['n']
sigma_sq_old = theta['sigma_sq']
# Regularization term
reg_term = 0
if reg and theta['theta_reg'] > 0:
for k in range(K):
for l in range(K):
reg_term += np.sum(np.square(np.absolute(A[k] - A[l])))
reg_term *= theta['theta_reg']
tasks = {}
for i, d in enumerate(img_data['dj']):
t = dict()
t['uuid'] = str(uuid.uuid4())
t['module'] = 'aitom.average.ml.faml.faml'
t['method'] = 'update_sigma_help'
t['kwargs'] = {
'img_db_path': img_data['db_path'],
'i': i,
'theta': theta,
'd': d,
'use_voronoi': use_voronoi
}
tasks[t['uuid']] = t
if len(sys.argv) > 1:
rt_s = [
_ for _ in TPRJB.run_iterator(
tasks,
redis_host=sys.argv[1],
redis_port=6379,
redis_password=
'******')
]
else:
rt_s = [
_ for _ in TPMU.run_iterator(tasks,
worker_num=MULTIPROCESSING_WORKER_NUM)
]
sigma_sq_new = 0
for rt in rt_s:
sigma_sq_new += rt['result']['result']
if reg:
sigma_sq_new += reg_term
return sigma_sq_new / (N * J)
def update_a(img_data, theta, alpha, use_voronoi, reg=True):
dj = img_data['dj']
compute_prob(img_data=img_data, theta=theta)
K = theta['K']
J = theta['J']
N = theta['N']
A_old = theta['A']
n = theta['n']
A_new = np.zeros([K, n, n, n], dtype=np.complex128)
# Regularization terms
reg_num = theta['theta_reg'] * np.sum(A_old, axis=0)
reg_denom = theta['theta_reg'] * K
tasks = {}
for k in range(K):
for i, d in enumerate(img_data['dj']):
t = {}
t['uuid'] = str(uuid.uuid4())
t['module'] = 'aitom.average.ml.faml.faml'
t['method'] = 'update_a_help'
t['kwargs'] = {
'img_db_path': img_data['db_path'],
'd': d,
'i': i,
'theta': theta,
'k': k,
'use_voronoi': use_voronoi
}
tasks[t['uuid']] = t
if len(sys.argv) > 1:
rt_s = [
_ for _ in TPRJB.run_iterator(
tasks,
redis_host=sys.argv[1],
redis_port=6379,
redis_password=
'******')
]
else:
rt_s = [
_ for _ in TPMU.run_iterator(tasks,
worker_num=MULTIPROCESSING_WORKER_NUM)
]
for rt in rt_s:
k = rt['result']['k']
A_new[k] += rt['result']['result']
for k in range(K):
if reg and theta['theta_reg'] > 0:
A_new[k] += reg_num
A_new[k] /= alpha[k] * N + reg_denom
else:
A_new[k] /= alpha[k] * N
return A_new
def compute_voronoi_weights(theta):
"""
Compute the weights of each configuration
by approximating the hypervolume of the voronoi region of each configuration
in the 6-dimensional space of all configurations
"""
N = theta['N']
K = theta['K']
n = theta['n']
theta['voronoi'] = [[None for _ in range(K)] for _ in range(N)]
tasks = {}
for k in range(K):
for i in range(N):
trans_list = theta['trans_list'][i][k]
t = dict()
t['uuid'] = str(uuid.uuid4())
t['module'] = 'aitom.geometry.volume.hypervolume.utils'
t['method'] = 'voronoi_weights_6d'
t['kwargs'] = {'phis': trans_list}
t['i'] = i
t['k'] = k
tasks[t['uuid']] = t
del i, k
if len(sys.argv) > 1:
rt_s = [
_ for _ in TPRJB.run_iterator(
tasks,
redis_host=sys.argv[1],
redis_port=6379,
redis_password=
'******')
]
else:
rt_s = [
_ for _ in TPMU.run_iterator(tasks,
worker_num=MULTIPROCESSING_WORKER_NUM)
]
for rt in rt_s:
i = tasks[rt['id']]['i']
k = tasks[rt['id']]['k']
theta['voronoi'][i][k] = rt['result']
