def __init__(self,
domain_factory,
shm_proxy,
lambdas=None,
nb_domains=os.cpu_count(),
ipc_notify=False):
super().__init__(domain_factory, lambdas, nb_domains, ipc_notify)
self._activations = [
mp.Value('b', False, lock=True) for i in range(nb_domains)
]
self._dones = [
mp.Value('b', False, lock=True) for i in range(nb_domains)
]
self._conditions = [mp.Condition() for i in range(nb_domains)]
self._shm_proxy = shm_proxy
self._shm_registers = {
} # Maps from registered method parameter types to vectorized array ranges
self._shm_types = {} # Maps from register index to type
self._shm_sizes = {
} # Maps from register method parameter types to number of arrays encoding each type
self._shm_arrays = [] # Methods' vectorized parameters
self._rsize = 0 # Total size of the register (updated below)
self._shm_lambdas = [None] * nb_domains # Vectorized lambdas' ids
self._shm_names = [None] * nb_domains # Vectorized methods' names
self._shm_params = [
None
] * nb_domains # Indices of methods' vectorized parameters
for i in range(nb_domains):
j = 0
for r in shm_proxy.register():
for k in range(r[1]):
m = shm_proxy.initialize(r[0])
if type(m) == list or type(m) == tuple:
if i == 0 and k == 0: # do it once for all the domains and redundant initializers
self._shm_sizes[r[0].__name__] = len(m)
self._shm_registers[r[0].__name__] = (
j, j + (r[1] * len(m)))
self._shm_types.update({
kk: r[0]
for kk in range(j, j + (r[1] * len(m)), len(m))
})
self._rsize += (r[1] * len(m))
self._shm_arrays.extend(m)
j += len(m)
else:
if i == 0 and k == 0: # do it once for all the domains and redundant initializers
self._shm_sizes[r[0].__name__] = 1
self._shm_registers[r[0].__name__] = (j, j + r[1])
self._shm_types.update(
{kk: r[0]
for kk in range(j, j + r[1])})
self._rsize += r[1]
self._shm_arrays.append(m)
j += 1
self._shm_lambdas[i] = mp.Value('i', -1, lock=True)
self._shm_names[i] = mp.Array('c', bytearray(100))
self._shm_params[i] = mp.Array(
'i', [-1] * sum(r[1] for r in shm_proxy.register()))
logger.info(rf'Using {nb_domains} parallel shared memory domains')
def __init__(self,
domain_factory,
lambdas=None,
nb_domains=os.cpu_count(),
ipc_notify=False):
self._domain_factory = domain_factory
self._lambdas = lambdas
self._active_domains = mp.Array('b',
[False for i in range(nb_domains)],
lock=True)
self._temp_connections = [None] * nb_domains
self._ipc_connections = [None] * nb_domains
self._processes = [None] * nb_domains
self._ipc_notify = ipc_notify
def initialize():
return mp.Array('i', [0, 0], lock=True)
def initialize():
return mp.Array('c', b'')
def initialize():
return mp.Array("b", [False, False, False, False], lock=True)
def initialize():
return mp.Array("c", b"")
def get_global_pval(self, hytest_method, node_to_leaves,
node_to_ancestral_nodes,
node_to_pwdist): #, leafpair_to_distance):
'''
Perform all inter-clusters' hypotheses tests
'''
if self.treeinfo_file_given < 1:
from ctypes import c_char_p
import os
print('\nPerforming {} tests...'.format(hytest_method))
# shared memory
lpd = self.leafpair_to_distance
max_node = max(node_to_leaves.keys())
if os.name == 'nt':
# windows
node_to_leaves_shared = [
node_to_leaves[n] if n in node_to_leaves.keys() else False
for n in range(max_node + 1)
]
else:
node_to_leaves_shared = [
mp.Array(c_char_p, node_to_leaves[n])
if n in node_to_leaves.keys() else False
for n in range(max_node + 1)
]
node_to_ancestral_nodes_shared = [
mp.Array('i', node_to_ancestral_nodes[n])
if n in node_to_ancestral_nodes else False
for n in range(max_node + 1)
]
node_to_pwdist_shared = [
mp.Array('d', node_to_pwdist[n])
if n in node_to_leaves.keys() else False
for n in range(max_node + 1)
]
# worker
def get_interclus_pval(np_list, ntl_dict, ntan_dict, ntpwd_dict,
q):
currp_np_to_pval = {}
for (i, j) in np_list:
if (ntan_dict[j] != False and i in list(ntan_dict[j])) or (
ntan_dict[i] != False and j in list(ntan_dict[i])):
pval = inter_cluster_hytest(
list(ntpwd_dict[i]),
list(ntpwd_dict[j])).hytest(hytest_method)
else:
ij_pwdist = sorted([
lpd[(x, y)] for x, y in itertools.combinations(
list(set(ntl_dict[i]) | set(ntl_dict[j])), 2)
])
# take the conservative (max) p-value comparing node i/j individually to i+j
pval = max([
inter_cluster_hytest(
list(ntpwd_dict[i]),
ij_pwdist).hytest(hytest_method),
inter_cluster_hytest(
list(ntpwd_dict[j]),
ij_pwdist).hytest(hytest_method)
])
currp_np_to_pval[(i, j)] = pval
q.put(currp_np_to_pval)
# multi-proc setup
manager = mp.Manager()
# shared memory
queue = manager.Queue()
# generate processes
processes = []
# split nodepair list into ncpu sets
nodepair_list = list(
itertools.combinations(node_to_leaves.keys(), 2))
shuffle(nodepair_list) # shuffle to make more equitable
increment = int(len(nodepair_list) / self.cores)
for p in range(self.cores):
if p == self.cores - 1:
curr_nodepair_list = nodepair_list[p * increment:]
else:
curr_nodepair_list = nodepair_list[p *
increment:(p *
increment) +
increment]
proc = mp.Process(target=get_interclus_pval,
args=(curr_nodepair_list,
node_to_leaves_shared,
node_to_ancestral_nodes_shared,
node_to_pwdist_shared, queue))
processes.append(proc)
proc.start()
# collect results to dictionary
for p in range(len(processes)):
self.nodepair_to_pval.update(queue.get())
# wait for all processes to end
for proc in processes:
proc.join()
if self.no_treeinfo == False:
print('Writing to treeinfo file...')
output = open(self.treeinfo_fname, 'a')
json.dump(self.remap_keys(self.nodepair_to_pval), output)
output.write('\n')
output.close()
"""
# single thread legacy code
nodepair_to_pval_single = {}
for i,j in itertools.combinations(node_to_leaves.keys(), 2):
if (j in node_to_ancestral_nodes and i in node_to_ancestral_nodes[j]) or (i in node_to_ancestral_nodes and j in node_to_ancestral_nodes[i]):
pval = inter_cluster_hytest(node_to_pwdist[i], node_to_pwdist[j]).hytest(hytest_method)
else:
ij_pwdist = sorted([leafpair_to_distance[(x,y)] for x,y in itertools.combinations(list(set(node_to_leaves[i])|set(node_to_leaves[j])), 2)])
# take the conservative (max) p-value comparing node i/j individually to i+j
pval = max([inter_cluster_hytest(node_to_pwdist[i], ij_pwdist).hytest(hytest_method), inter_cluster_hytest(node_to_pwdist[j], ij_pwdist).hytest(hytest_method)])
nodepair_to_pval_single[(i,j)] = pval
# check
print ('Sets of nodepair_to_pval: {}'.format(set(self.nodepair_to_pval.keys()) == set(nodepair_to_pval_single.keys())))
for (i, j), pval in self.nodepair_to_pval.items():
if pval != nodepair_to_pval_single[(i,j)]:
print (i, j, pval, nodepair_to_pval_single[(i, j)])
"""
return self.nodepair_to_pval
