def run_task_gevent(fun:object, fun_kwargs:list, processes:int=2, prefun:object=None, postfun:object=None):
size = len(fun_kwargs)
price = set_part(size)
pool = gPool(processes)
errorcount = 0
if prefun is not None:
pool.apply(prefun)
gevent_threads = []
for i, kwargs in enumerate(fun_kwargs):
res = pool.apply_async(fun, kwds = kwargs)
gevent_threads.append(res)
if i % price == 0:
pool.apply_async(complete_per, kwds={"s":size, "c":i})
pool.join()
results = []
for thread in gevent_threads:
if thread.successful():
results.append(thread.value)
else:
errorcount += 1
print("run gevent error in ", thread)
if postfun is not None:
pool.apply(postfun)
if errorcount > 0:
print("error count:", errorcount)
return results
def main():
pool = gPool(1000) #create thread pool with a limit of 1000
http_server = WSGIServer(('0.0.0.0', PORT), app,
spawn=pool) #create wsgi server
url = "http://localhost:" + str(PORT) + "/"
webbrowser.open(url, new=2) #Opens in new tab if possible
http_server.serve_forever()
def multigreenlet_map(func, *args, **kwargs):
"""
A method to parallelize mappable, IO-bound functions
"""
num_greenlets = kwargs.pop('num_greenlets', 10)
pool = gPool(num_greenlets)
result = pool.map(func, *args)
return result
def updata_user():
userCourses = User_course.query.all() #update the user's course info
for data in userCourses:
db.session.delete(data)
t1 = time()
allUser = User.query.all()
pool = gPool(120)
pool.map(updata, [[user.username,user.password_urp]
for user in allUser]
)
t2 = time()
return 'succeed \n run:%f'%(t2-t1)
gn = GeoNames(dbpath="./geoutils/GN_dump_20160407.sql")
end = time.clock()
print("Init time:{}".format(end - strt))
loclist = [
"estados Unidos", "India", "washington", "Madrid", "Paris", "hilton",
"chennai", "rio", "madras"
]
strt = time.clock()
s = [gn.query(l, min_popln=0) for l in loclist]
end = time.clock()
print("Normal serial processing: {}".format(end - strt))
exit(0)
gp = gPool(10)
print("gpool created")
tp = ThreadPool(10)
print("tpool created")
#mp = (10)
#print "mpool created"
def t(l):
return gn.query(l, min_popln=0)
##strt = time.clock()
##s = mp.map(t, loclist)
##end = time.clock()
##print "multiprocessing pool: {}".format(end-strt)
def echo(i):
time.sleep(0.001)
return i
# Non Deterministic Process Pool
from multiprocessing.pool import Pool as mPool
mp = mPool(10)
run1 = [a for a in mp.imap_unordered(echo, xrange(10))]
run2 = [a for a in mp.imap_unordered(echo, xrange(10))]
run3 = [a for a in mp.imap_unordered(echo, xrange(10))]
run4 = [a for a in mp.imap_unordered(echo, xrange(10))]
print run1 == run2 == run3 == run4
# Deterministic Gevent Pool
from gevent.pool import Pool as gPool
gp = gPool(10)
run1 = [a for a in gp.imap_unordered(echo, xrange(10))]
run2 = [a for a in gp.imap_unordered(echo, xrange(10))]
run3 = [a for a in gp.imap_unordered(echo, xrange(10))]
run4 = [a for a in gp.imap_unordered(echo, xrange(10))]
print run1 == run2 == run3 == run4