def __init__(self, domain_factory, shm_proxy, lambdas=None, nb_domains=os.cpu_count(), ipc_notify=False): super().__init__(domain_factory, lambdas, nb_domains, ipc_notify) self._activations = [ mp.Value('b', False, lock=True) for i in range(nb_domains) ] self._dones = [ mp.Value('b', False, lock=True) for i in range(nb_domains) ] self._conditions = [mp.Condition() for i in range(nb_domains)] self._shm_proxy = shm_proxy self._shm_registers = { } # Maps from registered method parameter types to vectorized array ranges self._shm_types = {} # Maps from register index to type self._shm_sizes = { } # Maps from register method parameter types to number of arrays encoding each type self._shm_arrays = [] # Methods' vectorized parameters self._rsize = 0 # Total size of the register (updated below) self._shm_lambdas = [None] * nb_domains # Vectorized lambdas' ids self._shm_names = [None] * nb_domains # Vectorized methods' names self._shm_params = [ None ] * nb_domains # Indices of methods' vectorized parameters for i in range(nb_domains): j = 0 for r in shm_proxy.register(): for k in range(r[1]): m = shm_proxy.initialize(r[0]) if type(m) == list or type(m) == tuple: if i == 0 and k == 0: # do it once for all the domains and redundant initializers self._shm_sizes[r[0].__name__] = len(m) self._shm_registers[r[0].__name__] = ( j, j + (r[1] * len(m))) self._shm_types.update({ kk: r[0] for kk in range(j, j + (r[1] * len(m)), len(m)) }) self._rsize += (r[1] * len(m)) self._shm_arrays.extend(m) j += len(m) else: if i == 0 and k == 0: # do it once for all the domains and redundant initializers self._shm_sizes[r[0].__name__] = 1 self._shm_registers[r[0].__name__] = (j, j + r[1]) self._shm_types.update( {kk: r[0] for kk in range(j, j + r[1])}) self._rsize += r[1] self._shm_arrays.append(m) j += 1 self._shm_lambdas[i] = mp.Value('i', -1, lock=True) self._shm_names[i] = mp.Array('c', bytearray(100)) self._shm_params[i] = mp.Array( 'i', [-1] * sum(r[1] for r in shm_proxy.register())) logger.info(rf'Using {nb_domains} parallel shared memory domains')
def __init__(self, domain_factory, lambdas=None, nb_domains=os.cpu_count(), ipc_notify=False): self._domain_factory = domain_factory self._lambdas = lambdas self._active_domains = mp.Array('b', [False for i in range(nb_domains)], lock=True) self._temp_connections = [None] * nb_domains self._ipc_connections = [None] * nb_domains self._processes = [None] * nb_domains self._ipc_notify = ipc_notify
def initialize(): return mp.Array('i', [0, 0], lock=True)
def initialize(): return mp.Array('c', b'')
def initialize(): return mp.Array("b", [False, False, False, False], lock=True)
def initialize(): return mp.Array("c", b"")
def get_global_pval(self, hytest_method, node_to_leaves, node_to_ancestral_nodes, node_to_pwdist): #, leafpair_to_distance): ''' Perform all inter-clusters' hypotheses tests ''' if self.treeinfo_file_given < 1: from ctypes import c_char_p import os print('\nPerforming {} tests...'.format(hytest_method)) # shared memory lpd = self.leafpair_to_distance max_node = max(node_to_leaves.keys()) if os.name == 'nt': # windows node_to_leaves_shared = [ node_to_leaves[n] if n in node_to_leaves.keys() else False for n in range(max_node + 1) ] else: node_to_leaves_shared = [ mp.Array(c_char_p, node_to_leaves[n]) if n in node_to_leaves.keys() else False for n in range(max_node + 1) ] node_to_ancestral_nodes_shared = [ mp.Array('i', node_to_ancestral_nodes[n]) if n in node_to_ancestral_nodes else False for n in range(max_node + 1) ] node_to_pwdist_shared = [ mp.Array('d', node_to_pwdist[n]) if n in node_to_leaves.keys() else False for n in range(max_node + 1) ] # worker def get_interclus_pval(np_list, ntl_dict, ntan_dict, ntpwd_dict, q): currp_np_to_pval = {} for (i, j) in np_list: if (ntan_dict[j] != False and i in list(ntan_dict[j])) or ( ntan_dict[i] != False and j in list(ntan_dict[i])): pval = inter_cluster_hytest( list(ntpwd_dict[i]), list(ntpwd_dict[j])).hytest(hytest_method) else: ij_pwdist = sorted([ lpd[(x, y)] for x, y in itertools.combinations( list(set(ntl_dict[i]) | set(ntl_dict[j])), 2) ]) # take the conservative (max) p-value comparing node i/j individually to i+j pval = max([ inter_cluster_hytest( list(ntpwd_dict[i]), ij_pwdist).hytest(hytest_method), inter_cluster_hytest( list(ntpwd_dict[j]), ij_pwdist).hytest(hytest_method) ]) currp_np_to_pval[(i, j)] = pval q.put(currp_np_to_pval) # multi-proc setup manager = mp.Manager() # shared memory queue = manager.Queue() # generate processes processes = [] # split nodepair list into ncpu sets nodepair_list = list( itertools.combinations(node_to_leaves.keys(), 2)) shuffle(nodepair_list) # shuffle to make more equitable increment = int(len(nodepair_list) / self.cores) for p in range(self.cores): if p == self.cores - 1: curr_nodepair_list = nodepair_list[p * increment:] else: curr_nodepair_list = nodepair_list[p * increment:(p * increment) + increment] proc = mp.Process(target=get_interclus_pval, args=(curr_nodepair_list, node_to_leaves_shared, node_to_ancestral_nodes_shared, node_to_pwdist_shared, queue)) processes.append(proc) proc.start() # collect results to dictionary for p in range(len(processes)): self.nodepair_to_pval.update(queue.get()) # wait for all processes to end for proc in processes: proc.join() if self.no_treeinfo == False: print('Writing to treeinfo file...') output = open(self.treeinfo_fname, 'a') json.dump(self.remap_keys(self.nodepair_to_pval), output) output.write('\n') output.close() """ # single thread legacy code nodepair_to_pval_single = {} for i,j in itertools.combinations(node_to_leaves.keys(), 2): if (j in node_to_ancestral_nodes and i in node_to_ancestral_nodes[j]) or (i in node_to_ancestral_nodes and j in node_to_ancestral_nodes[i]): pval = inter_cluster_hytest(node_to_pwdist[i], node_to_pwdist[j]).hytest(hytest_method) else: ij_pwdist = sorted([leafpair_to_distance[(x,y)] for x,y in itertools.combinations(list(set(node_to_leaves[i])|set(node_to_leaves[j])), 2)]) # take the conservative (max) p-value comparing node i/j individually to i+j pval = max([inter_cluster_hytest(node_to_pwdist[i], ij_pwdist).hytest(hytest_method), inter_cluster_hytest(node_to_pwdist[j], ij_pwdist).hytest(hytest_method)]) nodepair_to_pval_single[(i,j)] = pval # check print ('Sets of nodepair_to_pval: {}'.format(set(self.nodepair_to_pval.keys()) == set(nodepair_to_pval_single.keys()))) for (i, j), pval in self.nodepair_to_pval.items(): if pval != nodepair_to_pval_single[(i,j)]: print (i, j, pval, nodepair_to_pval_single[(i, j)]) """ return self.nodepair_to_pval