def processRequests():
global interrupted
from copy import copy
ctx = zmq.Context()
frontend = ctx.socket(zmq.ROUTER)
frontend.bind("tcp://*:6767")
backend = ctx.socket(zmq.DEALER)
backend.bind("tcp://*:6768")
workerCnt = 0
# Notice these workers connect before hand
poller = zmq.Poller()
poller.register(frontend, zmq.POLLIN)
poller.register(backend, zmq.POLLIN)
while True:
# Receive something from the frontend (requests)
# or the backend (completed work)
socks = dict(poller.poll(timeout = 1000))
if frontend in socks:
allParts = recvAll(frontend)
# Send something to a worker
sendAll(backend, allParts)
elif backend in socks:
# Get result of work from a worker
allParts = recvAll(backend)
# Give more work to the workers from the URLs we got back if any
#subParts = allParts[-1].split(";")
#for url in subParts[1:]:
# newWork = copy(allParts)
# newWork[-1] = url
# sendAll(backend, newWork)
#allParts[-1] = subParts[0]
# Receive parts back
sendAll(frontend, allParts)
if interrupted:
return
def __imgWorker(ctx, workerCnt):
""" I'm going to scrape some img tags from the URL I'm given"""
# context is the only thing thats threadsafe!!
sock = ctx.socket(zmq.REP)
sock.connect("tcp://localhost:6768")
print "New Worker started.."
print "Worker receiving"
allParts = zmqUtils.recvAll(sock)
clientId = allParts[0]
url = allParts[-1]
allParts[-1] += " from worker %i" % workerCnt
print "Received %s" % url
# Grab all the IMGs
(imgs, links) = htmlPuller.getAllImgTagsAndLinks(url)
allParts[-1] = imgs + "!-!" + links
zmqUtils.sendAll(sock, allParts)
print "WORKER DEAD!!!"
# Bind to the port specified in argv
frontend = context.socket(zmq.ROUTER)
print "Binding to port %i" % port
frontend.bind("tcp://*:%i" % port)
# A Router socket need is an asyncrhonous version of the REP
# socket. Instead of needing to serially process with recv/send/recv/send...
# A Router allows multiple simultaneous active requests.
#
# To quote from the zmq_socket man page:
# > When receiving messages a ZMQ_ROUTER socket shall prepend a message
# > part containing the identity of the originating peer to the message
# > before passing it to the application. Messages received are fair-queued
# > from among all connected peers. When sending messages a ZMQ_ROUTER socket
# > shall remove the first part of the message and use it to determine the
# > identity of the peer the message shall be routed to. If the peer does
# > not exist anymore the message shall be silently discarded.
#
# Below we wait for 2 requests before responding to the individual requests
while True:
print "Get 2 requests"
activeRequests = []
for i in range(0,2):
allParts = recvAll(frontend)
activeRequests.append(allParts)
print "Send 2 replies"
for parts in activeRequests:
print "Sending back %s" % repr(parts)
sendAll(frontend, parts)
for port in sys.argv[1:-1]:
sock.connect("tcp://localhost:%i" % int(port))
echoText = sys.argv[-1]
# A DEALER is an asynchronous REQ socket. Like a REQ socket it
# can connect to multiple hosts and will request from those
# hosts in a round-robin fashion. Unlike a REQ socket, the
# responses are not processed serially but fair queued.
#
# To quote the man page:
# > Each message sent is round-robinned among all connected
# > peers, and each message received is fair-queued from all
# > connected peers.
#
# Another important note, sometimes the REQ and DEALER sockets
# get stuck if the other side crashes and the REQ or DEALER is
# stuck, so implementing a timeout (see zmqDealerTimeoutWork.py)
# for more.
for i in itertools.count():
currEchoTxt = echoText + "(%i)" % i
print "Sending... %s" % currEchoTxt
# When connecting to a ZMQ_REP, an empty delimiter must be sent,
# See the zmq_socket man page for more info
sendAll(sock, ["", currEchoTxt])
#sock.send(currEchoTxt)
print "Receiving..."
echoTextBack = recvAll(sock)
print "Rcvd: > %s" % repr(echoTextBack)
waitExit()
