def MyParallelPool(nodes=None):
if nodes is None or nodes > 1:
p = ParallelPool(nodes)
try:
yield p
finally:
p.close()
p.join()
p.clear()
else:
#print("Using PseudoPool!")
yield PseudoPool()
def run_parallel_gray(gfile, qargs, hosts):
g_ = load_graph(gfile)
query_, cond_, _, _, _, _ = parse_args(qargs)
# Find query candidates
q_seed_ = list(query_.nodes())[0]
kl = Condition.get_node_label(query_, q_seed_)
kp = Condition.get_node_props(query_, q_seed_)
seeds = Condition.filter_nodes(g_, kl, kp) # Find all candidates
if not seeds: ## No seed candidates
print("No more seed vertices available. Exit G-Ray algorithm.")
return
# Split seed list
num_seeds = len(seeds)
num_hosts = len(hosts)
num_members = num_seeds / num_hosts
seed_lists = list()
for i in range(num_hosts):
st = i * num_members
ed = num_seeds if (i == num_hosts - 1) else (i + 1) * num_members
seed_lists.append(seeds[st:ed])
servers = tuple([":".join([addr, port]) for addr in hosts])
st = time.time()
pool = Pool(1, servers=servers)
ret = pool.amap(
partial(process_multiple_gray,
g_file=gfile,
q_seed=q_seed_,
q_args=qargs), seed_lists)
print(ret.get())
pool.close()
pool.join()
ed = time.time()
print("Parallel G-Ray time: %f" % (ed - st))
word_count[word] = 1
return word_count
def top_ten(wordcount):
for key, value in sorted(word_count.items(), key=operator.itemgetter(1), reverse=True)[:10]:
print key, value
def wordcount(f):
words = word_list(f)
clean = cleanup(words)
word_dict(clean)
if __name__ == '__main__':
info('Mainline function')
pool.ncpus = 2
pool.servers = ('localhost:17320',)
words = pool.map(m_word_list, f)
pool.join()
pool.servers = ('localhost:17320',)
clean_list = pool.map(m_cleanup, words)
pool.join(clean_list, )
pool.servers = ('localhost:17320',)
word_count = pool.map(word_dict, cleanlist)
pool.join(word_count, )
top_ten(word_count)