def processRequests(): global interrupted from copy import copy ctx = zmq.Context() frontend = ctx.socket(zmq.ROUTER) frontend.bind("tcp://*:6767") backend = ctx.socket(zmq.DEALER) backend.bind("tcp://*:6768") workerCnt = 0 # Notice these workers connect before hand poller = zmq.Poller() poller.register(frontend, zmq.POLLIN) poller.register(backend, zmq.POLLIN) while True: # Receive something from the frontend (requests) # or the backend (completed work) socks = dict(poller.poll(timeout = 1000)) if frontend in socks: allParts = recvAll(frontend) # Send something to a worker sendAll(backend, allParts) elif backend in socks: # Get result of work from a worker allParts = recvAll(backend) # Give more work to the workers from the URLs we got back if any #subParts = allParts[-1].split(";") #for url in subParts[1:]: # newWork = copy(allParts) # newWork[-1] = url # sendAll(backend, newWork) #allParts[-1] = subParts[0] # Receive parts back sendAll(frontend, allParts) if interrupted: return
def __imgWorker(ctx, workerCnt): """ I'm going to scrape some img tags from the URL I'm given""" # context is the only thing thats threadsafe!! sock = ctx.socket(zmq.REP) sock.connect("tcp://localhost:6768") print "New Worker started.." print "Worker receiving" allParts = zmqUtils.recvAll(sock) clientId = allParts[0] url = allParts[-1] allParts[-1] += " from worker %i" % workerCnt print "Received %s" % url # Grab all the IMGs (imgs, links) = htmlPuller.getAllImgTagsAndLinks(url) allParts[-1] = imgs + "!-!" + links zmqUtils.sendAll(sock, allParts) print "WORKER DEAD!!!"
# Bind to the port specified in argv frontend = context.socket(zmq.ROUTER) print "Binding to port %i" % port frontend.bind("tcp://*:%i" % port) # A Router socket need is an asyncrhonous version of the REP # socket. Instead of needing to serially process with recv/send/recv/send... # A Router allows multiple simultaneous active requests. # # To quote from the zmq_socket man page: # > When receiving messages a ZMQ_ROUTER socket shall prepend a message # > part containing the identity of the originating peer to the message # > before passing it to the application. Messages received are fair-queued # > from among all connected peers. When sending messages a ZMQ_ROUTER socket # > shall remove the first part of the message and use it to determine the # > identity of the peer the message shall be routed to. If the peer does # > not exist anymore the message shall be silently discarded. # # Below we wait for 2 requests before responding to the individual requests while True: print "Get 2 requests" activeRequests = [] for i in range(0,2): allParts = recvAll(frontend) activeRequests.append(allParts) print "Send 2 replies" for parts in activeRequests: print "Sending back %s" % repr(parts) sendAll(frontend, parts)
for port in sys.argv[1:-1]: sock.connect("tcp://localhost:%i" % int(port)) echoText = sys.argv[-1] # A DEALER is an asynchronous REQ socket. Like a REQ socket it # can connect to multiple hosts and will request from those # hosts in a round-robin fashion. Unlike a REQ socket, the # responses are not processed serially but fair queued. # # To quote the man page: # > Each message sent is round-robinned among all connected # > peers, and each message received is fair-queued from all # > connected peers. # # Another important note, sometimes the REQ and DEALER sockets # get stuck if the other side crashes and the REQ or DEALER is # stuck, so implementing a timeout (see zmqDealerTimeoutWork.py) # for more. for i in itertools.count(): currEchoTxt = echoText + "(%i)" % i print "Sending... %s" % currEchoTxt # When connecting to a ZMQ_REP, an empty delimiter must be sent, # See the zmq_socket man page for more info sendAll(sock, ["", currEchoTxt]) #sock.send(currEchoTxt) print "Receiving..." echoTextBack = recvAll(sock) print "Rcvd: > %s" % repr(echoTextBack) waitExit()