def MyParallelPool(nodes=None): if nodes is None or nodes > 1: p = ParallelPool(nodes) try: yield p finally: p.close() p.join() p.clear() else: #print("Using PseudoPool!") yield PseudoPool()
def run_parallel_gray(gfile, qargs, hosts): g_ = load_graph(gfile) query_, cond_, _, _, _, _ = parse_args(qargs) # Find query candidates q_seed_ = list(query_.nodes())[0] kl = Condition.get_node_label(query_, q_seed_) kp = Condition.get_node_props(query_, q_seed_) seeds = Condition.filter_nodes(g_, kl, kp) # Find all candidates if not seeds: ## No seed candidates print("No more seed vertices available. Exit G-Ray algorithm.") return # Split seed list num_seeds = len(seeds) num_hosts = len(hosts) num_members = num_seeds / num_hosts seed_lists = list() for i in range(num_hosts): st = i * num_members ed = num_seeds if (i == num_hosts - 1) else (i + 1) * num_members seed_lists.append(seeds[st:ed]) servers = tuple([":".join([addr, port]) for addr in hosts]) st = time.time() pool = Pool(1, servers=servers) ret = pool.amap( partial(process_multiple_gray, g_file=gfile, q_seed=q_seed_, q_args=qargs), seed_lists) print(ret.get()) pool.close() pool.join() ed = time.time() print("Parallel G-Ray time: %f" % (ed - st))
#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 1997-2016 California Institute of Technology. # Copyright (c) 2016-2018 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/pathos/blob/master/LICENSE from pathos.parallel import stats from pathos.parallel import ParallelPool as Pool pool = Pool() def host(id): import socket import time time.sleep(1.0) return "Rank: %d -- %s" % (id, socket.gethostname()) print("Evaluate 10 items on 2 cpus") #FIXME: reset lport below pool.ncpus = 2 pool.servers = ('localhost:5653', ) res5 = pool.map(host, range(10)) print(pool) print('\n'.join(res5)) print(stats()) print('') # end of file
#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 1997-2016 California Institute of Technology. # License: 3-clause BSD. The full license text is available at: # - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pathos/LICENSE from pathos.parallel import stats from pathos.parallel import ParallelPool as Pool pool = Pool() def host(id): import socket return "Rank: %d -- %s" % (id, socket.gethostname()) print "Evaluate 10 items on 1 cpu" pool.ncpus = 1 res3 = pool.map(host, range(10)) print pool print '\n'.join(res3) print stats() print "Evaluate 10 items on 2 cpus" pool.ncpus = 2 res5 = pool.map(host, range(10)) print pool print '\n'.join(res5) print stats() # end of file
#!/usr/bin/python import os import urllib import operator import timing from pathos.parallel import ParallelPool as Pool pool = Pool() #url = 'http://www.gutenberg.org/cache/epub/25990/pg25990.txt' #f = urllib.urlopen(url) manager_address = 'localhost' f = open('/root/homework/pg25990.txt') def info(title): print title print 'module name:', __name__ if hasattr(os, 'getppid'): print 'parent process:', os.getppid() print 'process id:', os.getpid() @timing.timed def m_word_list(file): info('server word_list function') words = [] words.append(f.read().split()) # print words
#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 1997-2016 California Institute of Technology. # Copyright (c) 2016-2022 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/pathos/blob/master/LICENSE from pathos.parallel import stats from pathos.parallel import ParallelPool as Pool pool = Pool() def host(id): import socket return "Rank: %d -- %s" % (id, socket.gethostname()) print("Evaluate 10 items on 1 cpu") pool.ncpus = 1 res3 = pool.map(host, range(10)) print(pool) print('\n'.join(res3)) print(stats()) print("Evaluate 10 items on 2 cpus") pool.ncpus = 2 res5 = pool.map(host, range(10)) print(pool) print('\n'.join(res5)) print(stats())
''' Created on 30 Aug 2020 @author: snake91 ''' from pathos.parallel import ParallelPool as Pool import time pool = Pool() def host(idhost): import socket from math import sin sumhost = 0 for i in range(1, 1000): for j in range(1, 10000): sumhost = sumhost + sin(i / j) return "Rank: %d -- %s. %f" % (idhost, socket.gethostname(), sumhost + idhost) pool.ncpus = 20 print("Evaluate on " + str(pool.ncpus) + " cpus") pool.servers = ('192.168.1.113:5653', '192.168.1.109:5654') res5 = pool.amap(host, range(16)) while not res5.ready(): time.sleep(1)
#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 1997-2016 California Institute of Technology. # Copyright (c) 2016-2019 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/pathos/blob/master/LICENSE from pathos.parallel import stats from pathos.parallel import ParallelPool as Pool pool = Pool() def host(id): import socket import time time.sleep(1.0) return "Rank: %d -- %s" % (id, socket.gethostname()) print("Evaluate 10 items on 2 cpus") #FIXME: reset lport below pool.ncpus = 2 pool.servers = ('localhost:5653',) res5 = pool.map(host, range(10)) print(pool) print('\n'.join(res5)) print(stats()) print('') # end of file
from functools import partial sys.path.append(".") from patternmatching.gray.parallel.query_call import parse_args from patternmatching.gray import rwr, extract from patternmatching.query.Condition import * # https://stackoverflow.com/questions/26876898/python-multiprocessing-with-distributed-cluster/26948258 def sleepy_squared(x): from time import sleep sleep(0.5) return x**2 p = Pool(4) res = p.amap(sleepy_squared, range(10)) print(res.get()) ################ port = "5000" def load_graph(graph_json): with open(graph_json, "r") as f: json_data = json.load(f) graph = json_graph.node_link_graph(json_data) numv = graph.number_of_nodes() nume = graph.number_of_edges() print("Input Graph: " + str(numv) + " vertices, " + str(nume) + " edges")
from pathos.parallel import ParallelPool as Pool pool = Pool() def host(id): import socket import time time.sleep(1.0) return "Rank: %d -- %s" % (id, socket.gethostname()) print("Evaluate 10 items on 2 cpus") #FIXME: reset lport below pool.ncpus = 2 pool.servers = ('56v6f22-l:55577', ) res5 = pool.map(host, range(10)) print(pool) print('\n'.join(res5)) # print(stats()) print('') # end of file