def MyParallelPool(nodes=None): if nodes is None or nodes > 1: p = ParallelPool(nodes) try: yield p finally: p.close() p.join() p.clear() else: #print("Using PseudoPool!") yield PseudoPool()
def run_parallel_gray(gfile, qargs, hosts): g_ = load_graph(gfile) query_, cond_, _, _, _, _ = parse_args(qargs) # Find query candidates q_seed_ = list(query_.nodes())[0] kl = Condition.get_node_label(query_, q_seed_) kp = Condition.get_node_props(query_, q_seed_) seeds = Condition.filter_nodes(g_, kl, kp) # Find all candidates if not seeds: ## No seed candidates print("No more seed vertices available. Exit G-Ray algorithm.") return # Split seed list num_seeds = len(seeds) num_hosts = len(hosts) num_members = num_seeds / num_hosts seed_lists = list() for i in range(num_hosts): st = i * num_members ed = num_seeds if (i == num_hosts - 1) else (i + 1) * num_members seed_lists.append(seeds[st:ed]) servers = tuple([":".join([addr, port]) for addr in hosts]) st = time.time() pool = Pool(1, servers=servers) ret = pool.amap( partial(process_multiple_gray, g_file=gfile, q_seed=q_seed_, q_args=qargs), seed_lists) print(ret.get()) pool.close() pool.join() ed = time.time() print("Parallel G-Ray time: %f" % (ed - st))
word_count[word] = 1 return word_count def top_ten(wordcount): for key, value in sorted(word_count.items(), key=operator.itemgetter(1), reverse=True)[:10]: print key, value def wordcount(f): words = word_list(f) clean = cleanup(words) word_dict(clean) if __name__ == '__main__': info('Mainline function') pool.ncpus = 2 pool.servers = ('localhost:17320',) words = pool.map(m_word_list, f) pool.join() pool.servers = ('localhost:17320',) clean_list = pool.map(m_cleanup, words) pool.join(clean_list, ) pool.servers = ('localhost:17320',) word_count = pool.map(word_dict, cleanlist) pool.join(word_count, ) top_ten(word_count)