def MyParallelPool(nodes=None): if nodes is None or nodes > 1: p = ParallelPool(nodes) try: yield p finally: p.close() p.join() p.clear() else: #print("Using PseudoPool!") yield PseudoPool()
def run_parallel_gray(gfile, qargs, hosts): g_ = load_graph(gfile) query_, cond_, _, _, _, _ = parse_args(qargs) # Find query candidates q_seed_ = list(query_.nodes())[0] kl = Condition.get_node_label(query_, q_seed_) kp = Condition.get_node_props(query_, q_seed_) seeds = Condition.filter_nodes(g_, kl, kp) # Find all candidates if not seeds: ## No seed candidates print("No more seed vertices available. Exit G-Ray algorithm.") return # Split seed list num_seeds = len(seeds) num_hosts = len(hosts) num_members = num_seeds / num_hosts seed_lists = list() for i in range(num_hosts): st = i * num_members ed = num_seeds if (i == num_hosts - 1) else (i + 1) * num_members seed_lists.append(seeds[st:ed]) servers = tuple([":".join([addr, port]) for addr in hosts]) st = time.time() pool = Pool(1, servers=servers) ret = pool.amap( partial(process_multiple_gray, g_file=gfile, q_seed=q_seed_, q_args=qargs), seed_lists) print(ret.get()) pool.close() pool.join() ed = time.time() print("Parallel G-Ray time: %f" % (ed - st))