def MyParallelPool(nodes=None):
if nodes is None or nodes > 1:
p = ParallelPool(nodes)
try:
yield p
finally:
p.close()
p.join()
p.clear()
else:
#print("Using PseudoPool!")
yield PseudoPool()
def run_parallel_gray(gfile, qargs, hosts):
g_ = load_graph(gfile)
query_, cond_, _, _, _, _ = parse_args(qargs)
# Find query candidates
q_seed_ = list(query_.nodes())[0]
kl = Condition.get_node_label(query_, q_seed_)
kp = Condition.get_node_props(query_, q_seed_)
seeds = Condition.filter_nodes(g_, kl, kp) # Find all candidates
if not seeds: ## No seed candidates
print("No more seed vertices available. Exit G-Ray algorithm.")
return
# Split seed list
num_seeds = len(seeds)
num_hosts = len(hosts)
num_members = num_seeds / num_hosts
seed_lists = list()
for i in range(num_hosts):
st = i * num_members
ed = num_seeds if (i == num_hosts - 1) else (i + 1) * num_members
seed_lists.append(seeds[st:ed])
servers = tuple([":".join([addr, port]) for addr in hosts])
st = time.time()
pool = Pool(1, servers=servers)
ret = pool.amap(
partial(process_multiple_gray,
g_file=gfile,
q_seed=q_seed_,
q_args=qargs), seed_lists)
print(ret.get())
pool.close()
pool.join()
ed = time.time()
print("Parallel G-Ray time: %f" % (ed - st))
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2018 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pathos/blob/master/LICENSE
from pathos.parallel import stats
from pathos.parallel import ParallelPool as Pool
pool = Pool()
def host(id):
import socket
import time
time.sleep(1.0)
return "Rank: %d -- %s" % (id, socket.gethostname())
print("Evaluate 10 items on 2 cpus") #FIXME: reset lport below
pool.ncpus = 2
pool.servers = ('localhost:5653', )
res5 = pool.map(host, range(10))
print(pool)
print('\n'.join(res5))
print(stats())
print('')
# end of file
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pathos/LICENSE
from pathos.parallel import stats
from pathos.parallel import ParallelPool as Pool
pool = Pool()
def host(id):
import socket
return "Rank: %d -- %s" % (id, socket.gethostname())
print "Evaluate 10 items on 1 cpu"
pool.ncpus = 1
res3 = pool.map(host, range(10))
print pool
print '\n'.join(res3)
print stats()
print "Evaluate 10 items on 2 cpus"
pool.ncpus = 2
res5 = pool.map(host, range(10))
print pool
print '\n'.join(res5)
print stats()
# end of file
#!/usr/bin/python
import os
import urllib
import operator
import timing
from pathos.parallel import ParallelPool as Pool
pool = Pool()
#url = 'http://www.gutenberg.org/cache/epub/25990/pg25990.txt'
#f = urllib.urlopen(url)
manager_address = 'localhost'
f = open('/root/homework/pg25990.txt')
def info(title):
print title
print 'module name:', __name__
if hasattr(os, 'getppid'):
print 'parent process:', os.getppid()
print 'process id:', os.getpid()
@timing.timed
def m_word_list(file):
info('server word_list function')
words = []
words.append(f.read().split())
# print words
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2022 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pathos/blob/master/LICENSE
from pathos.parallel import stats
from pathos.parallel import ParallelPool as Pool
pool = Pool()
def host(id):
import socket
return "Rank: %d -- %s" % (id, socket.gethostname())
print("Evaluate 10 items on 1 cpu")
pool.ncpus = 1
res3 = pool.map(host, range(10))
print(pool)
print('\n'.join(res3))
print(stats())
print("Evaluate 10 items on 2 cpus")
pool.ncpus = 2
res5 = pool.map(host, range(10))
print(pool)
print('\n'.join(res5))
print(stats())
'''
Created on 30 Aug 2020
@author: snake91
'''
from pathos.parallel import ParallelPool as Pool
import time
pool = Pool()
def host(idhost):
import socket
from math import sin
sumhost = 0
for i in range(1, 1000):
for j in range(1, 10000):
sumhost = sumhost + sin(i / j)
return "Rank: %d -- %s. %f" % (idhost, socket.gethostname(),
sumhost + idhost)
pool.ncpus = 20
print("Evaluate on " + str(pool.ncpus) + " cpus")
pool.servers = ('192.168.1.113:5653', '192.168.1.109:5654')
res5 = pool.amap(host, range(16))
while not res5.ready():
time.sleep(1)
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2019 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pathos/blob/master/LICENSE
from pathos.parallel import stats
from pathos.parallel import ParallelPool as Pool
pool = Pool()
def host(id):
import socket
import time
time.sleep(1.0)
return "Rank: %d -- %s" % (id, socket.gethostname())
print("Evaluate 10 items on 2 cpus") #FIXME: reset lport below
pool.ncpus = 2
pool.servers = ('localhost:5653',)
res5 = pool.map(host, range(10))
print(pool)
print('\n'.join(res5))
print(stats())
print('')
# end of file
from functools import partial
sys.path.append(".")
from patternmatching.gray.parallel.query_call import parse_args
from patternmatching.gray import rwr, extract
from patternmatching.query.Condition import *
# https://stackoverflow.com/questions/26876898/python-multiprocessing-with-distributed-cluster/26948258
def sleepy_squared(x):
from time import sleep
sleep(0.5)
return x**2
p = Pool(4)
res = p.amap(sleepy_squared, range(10))
print(res.get())
################
port = "5000"
def load_graph(graph_json):
with open(graph_json, "r") as f:
json_data = json.load(f)
graph = json_graph.node_link_graph(json_data)
numv = graph.number_of_nodes()
nume = graph.number_of_edges()
print("Input Graph: " + str(numv) + " vertices, " + str(nume) + " edges")
from pathos.parallel import ParallelPool as Pool
pool = Pool()
def host(id):
import socket
import time
time.sleep(1.0)
return "Rank: %d -- %s" % (id, socket.gethostname())
print("Evaluate 10 items on 2 cpus") #FIXME: reset lport below
pool.ncpus = 2
pool.servers = ('56v6f22-l:55577', )
res5 = pool.map(host, range(10))
print(pool)
print('\n'.join(res5))
# print(stats())
print('')
# end of file