def start_mapper(problem, modelargs):
import sys
import multiprocessing
import subprocess
from amqp_map.config import SERVICE_HOST
from amqp_map.core import connect, Mapper
server = connect(SERVICE_HOST)
mapper = Mapper(server, "bumps")
cpus = multiprocessing.cpu_count()
pipes = []
for _ in range(cpus):
cmd = [sys.argv[0], "--worker"] + modelargs
# print "starting",sys.argv[0],"in",os.getcwd(),"with",cmd
pipe = subprocess.Popen(cmd, universal_newlines=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
pipes.append(pipe)
for pipe in pipes:
if pipe.poll() > 0:
raise RuntimeError("subprocess returned %d\nout: %s\nerr: %s"
% (pipe.returncode, pipe.stdout, pipe.stderr))
#os.system(" ".join(cmd+["&"]))
import atexit
def exit_fun():
for p in pipes:
p.terminate()
atexit.register(exit_fun)
# print "returning mapper",mapper
return mapper
def start_mapper(problem, modelargs, cpus=0):
import sys
import multiprocessing
import subprocess
from amqp_map.config import SERVICE_HOST
from amqp_map.core import connect, Mapper
server = connect(SERVICE_HOST)
mapper = Mapper(server, "bumps")
cpus = multiprocessing.cpu_count()
pipes = []
for _ in range(cpus):
cmd = [sys.argv[0], "--worker"] + modelargs
# print "starting",sys.argv[0],"in",os.getcwd(),"with",cmd
pipe = subprocess.Popen(cmd,
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
pipes.append(pipe)
for pipe in pipes:
if pipe.poll() > 0:
raise RuntimeError("subprocess returned %d\nout: %s\nerr: %s" %
(pipe.returncode, pipe.stdout, pipe.stderr))
#os.system(" ".join(cmd+["&"]))
import atexit
def exit_fun():
for p in pipes:
p.terminate()
atexit.register(exit_fun)
# print "returning mapper",mapper
return mapper
def start_worker(problem):
#sys.stderr = open("bumps-%d.log"%os.getpid(),"w")
#print >>sys.stderr,"worker is starting"; sys.stdout.flush()
from amqp_map.config import SERVICE_HOST
from amqp_map.core import connect, start_worker as serve
server = connect(SERVICE_HOST)
#os.system("echo 'serving' > /tmp/map.%d"%(os.getpid()))
# print "worker is serving"; sys.stdout.flush()
serve(server, "bumps", problem.nllf)
def start_worker(problem):
#sys.stderr = open("bumps-%d.log"%os.getpid(),"w")
#print >>sys.stderr,"worker is starting"; sys.stdout.flush()
from amqp_map.config import SERVICE_HOST
from amqp_map.core import connect, start_worker as serve
server = connect(SERVICE_HOST)
#os.system("echo 'serving' > /tmp/map.%d"%(os.getpid()))
# print "worker is serving"; sys.stdout.flush()
serve(server, "bumps", problem.nllf)
import time import numpy as np from amqp_map.config import SERVICE_HOST from amqp_map.core import connect, Mapper server = connect(SERVICE_HOST) square = Mapper(server, "square") #print square(xrange(5,10)) #for i,v in square.async(xrange(-20,-15)): print i,v t0 = time.time() n=10000 print "start direct",n [ x*x for x in xrange(n)] print "direct time",1000*(time.time()-t0)/n,"ms/call" t0 = time.time() n=100 print "start big",n square([x*np.ones(3) for x in xrange(n)]) print "remote time",1000*(time.time()-t0)/n,"ms/call"
#!/usr/bin/env python
import time
from amqp_map.config import SERVICE_HOST
from amqp_map.core import connect
from amqp_map.rpc import RPC
rpc = RPC(connect(SERVICE_HOST))
for item in sys.argv[1:]:
print rpc.echo.echo(item)