def recv(self):
"""
Returns the reply message or None if there was no reply.
"""
try:
items = self.poller.poll(self.timeout)
except KeyboardInterrupt:
return # interrupted
if items:
# if we got a reply, process it
msg = self.client.recv_multipart()
if self.verbose:
logging.info("I: received reply:")
dump(msg)
# Don't try to handle errors, just assert noisily
assert len(msg) >= 4
empty = msg.pop(0)
header = msg.pop(0)
assert MDP.C_CLIENT == header
service = msg.pop(0)
return msg
else:
logging.warn("W: permanent error, abandoning request")
def handle_snapshot(self, msg):
"""snapshot requests"""
logging.info("I: received state request on snapshot: %s" % msg)
if len(msg) != 3 or msg[1] != "ICANHAZ?":
print "E: bad request, aborting"
dump(msg)
self.loop.stop()
return
identity, request, subtree = msg
if subtree:
# Send state snapshot to client
route = Route(self.snapshot, identity, subtree)
# For each entry in kvmap, send kvmsg to client
for k, v in self.kvmap.items():
logging.info("I: Sending snapshot: %s" % v)
send_single(k, v, route)
# Now send END message with sequence number
self.snapshot.send(identity, zmq.SNDMORE)
kvmsg = KVMsg(self.sequence)
kvmsg.key = "KTHXBAI"
kvmsg.body = subtree
logging.info("I: Sending snapshot end message: %s" % kvmsg)
kvmsg.send(self.snapshot)
def mediate(self):
"""Main broker work happens here"""
while True:
try:
items = self.poller.poll(self.HEARTBEAT_INTERVAL)
except KeyboardInterrupt:
break # Interrupted
if items:
msg = self.socket.recv_multipart()
if self.verbose:
logging.info("I: received message:")
dump(msg)
sender = msg.pop(0)
empty = msg.pop(0)
assert empty == ''
header = msg.pop(0)
if (MDP.C_CLIENT == header):
self.process_client(sender, msg)
elif (MDP.W_WORKER == header):
self.process_worker(sender, msg)
else:
logging.error("E: invalid message:")
dump(msg)
self.purge_workers()
self.send_heartbeats()
def mediate(self):#Main SBROKER work happens here, TEST ALL INCOMING MESSAGES FOR VALIDITY
while True:# LOOP
try:
items = self.poller.poll(self.HEARTBEAT_INTERVAL)# SET POLLING INTERVAL ON SOCKET
except KeyboardInterrupt:#STOP LOOP IF CTRL+C IS PRESSED
break# Interrupted
if items:# IF DATA CAME IN ON POLLING SOCKET
msg = self.socket.recv_multipart()# ASSIGN INCOMING MESSAGE TO 'MSG' LIST VARIABLE
if self.verbose:
logging.info("I: received message:")
dump(msg)
sender = msg.pop(0)# ITEM.0 IN MSG IDENTIFIES THE CWRITER
empty = msg.pop(0)# ITEM.1 IN MSG IDENTIFIES EMPTY DELIMITER
assert empty == ''# CHECK IF EMPTY DELIMITER IS ACTUALLY EMPTY
header = msg.pop(0)# ITEM.2 IN MSG IDENTIFIES CWRITER THAT SENT THE MESSAGE
if (MDP.C_CLIENT == header):# CHECK MDP PROTOCOL AGAINST CWRITER TO CHECK THAT CWRITER LINKED TO SOCKET
self.process_cwriter(sender, msg)# START PROCESSING THE CWRITER, RUN PROCESSING FUNCTION ON IT
elif (MDP.W_WORKER == header):# CHECK MDP PROTOCOL AGAINST CREADER TO CHECK THAT CREADER LINKED TO SOCKET
self.process_creader(sender, msg)# START PROCESSING THE CWRITER, RUN PROCESSING FUNCTION ON IT
else:# INVALIDATE MESSAGE IF IT DIN'T COME FROM A CREADER OR A CWRITER, OR WAS INVALIDLY CONSTRUCTED
logging.error("E: invalid message:")
dump(msg)
self.purge_creaders()# RUN FUNCTION TO CHECK FOR REMOVED/DISCONNECTED CREADERS AND DELETE THEM AND THEIR SESSIONS
self.send_heartbeats()# RUN FUNCTION FOR SENDING HEARTBEATS TO CREADERS THAT ARE IN ACTIVE SESSIONS LIST
def main():
verbose = '-v' in sys.argv
ctx = zmq.Context()
# Prepare server socket with predictable identity
bind_endpoint = "tcp://*:5555"
connect_endpoint = "tcp://localhost:5555"
server = ctx.socket(zmq.ROUTER)
server.identity = connect_endpoint
server.bind(bind_endpoint)
print "I: service is ready at", bind_endpoint
while True:
try:
request = server.recv_multipart()
except:
break # Interrupted
# Frame 0: identity of client
# Frame 1: PING, or client control frame
# Frame 2: request body
address, control = request[:2]
reply = [address, control]
if control == "PING":
reply[1] = "PONG"
else:
reply.append("OK")
if verbose:
dump(reply)
server.send_multipart(reply)
print "W: interrupted"
def main():
verbose = '-v' in sys.argv
ctx = zmq.Context()
# Prepare server socket with predictable identity
bind_endpoint = "tcp://*:5555"
connect_endpoint = "tcp://localhost:5555"
server = ctx.socket(zmq.ROUTER)
server.identity = connect_endpoint
server.bind(bind_endpoint)
print "I: service is ready at", bind_endpoint
while True:
try:
request = server.recv_multipart()
except:
break # Interrupted
# Frame 0: identity of client
# Frame 1: PING, or client control frame
# Frame 2: request body
address, control = request[:2]
reply = [address, control]
if control == "PING":
reply[1] = "PONG"
else:
reply.append("OK")
if verbose:
dump(reply)
server.send_multipart(reply)
print "W: interrupted"
def mediate(self):
"""Main broker work happens here"""
while True:
try:
items = self.poller.poll(self.HEARTBEAT_INTERVAL)
except KeyboardInterrupt:
break # Interrupted
if items:
msg = self.socket.recv_multipart()
if self.verbose:
logging.info("I: received message:")
dump(msg)
sender = msg.pop(0)
empty = msg.pop(0)
assert empty == b''
header = msg.pop(0)
if (MDP.C_CLIENT == header):
self.process_client(sender, msg)
elif (MDP.W_WORKER == header):
self.process_worker(sender, msg)
else:
logging.error("E: invalid message:")
dump(msg)
self.purge_workers()
self.send_heartbeats()
def recv(self):
"""Returns the reply message or None if there was no reply."""
try:
items = self.poller.poll(self.timeout)
except KeyboardInterrupt:
return # interrupted
if items:
# if we got a reply, process it
msg = self.client.recv_multipart()
if self.verbose:
logging.info("I: received reply:")
dump(msg)
# Don't try to handle errors, just assert noisily
assert len(msg) >= 4
empty = msg.pop(0)
header = msg.pop(0)
assert MDP.C_CLIENT == header
service = msg.pop(0)
return msg
else:
logging.warn("W: permanent error, abandoning request")
def listener_thread(pipe):
# Print everything that arrives on pipe
while True:
try:
dump(pipe.recv_multipart())
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
break # Interrupted
def send(self, service, request):
if not isinstance(request, list):
request = [request]
# Prefix request with protocol frames
# Frame 0: empty (REQ emulation)
# Frame 1: "MDPCxy" (six bytes, MDP/Client x.y)
# Frame 2: Service name (printable string)
request = ['', MDP.C_CLIENT, service] + request
if self.verbose:
logging.warn("I: send request to '%s' service: ", service)
dump(request)
self.client.send_multipart(request)
def send_to_sbroker(self, command, option=None, msg=None):# FUNCTION IS CALLED FOR SENDING MESSAGES TO SBROKER
if msg is None:# CHECK IF MESSAGE IS PROVIED TO THE FUNCTION
msg = []# IF NOT THEN SET MESSAGE TO EMPTY LIST
elif not isinstance(msg, list):# CHECK THAT MESSAGE THAT HAS BEEN PROVIED IS TYPE LIST
msg = [msg]# IF NOT THEN MAKE PROVIED MESSAGE INTO A LIST
if option:# IF OPTION IS PROVIED THEN REWRAP THE MESSAGE BY PLACING OPTION FIRST:
msg = [option] + msg# NOW THE MESSAGE LOOKS LIKE THIS: [[OPTION],[MESSAGE]]
msg = ['', MDP.W_WORKER, command] + msg# REWRAP THE MESSAGE AGAIN WITH EMPTY DELIMITER, CREADER ID AND COMMAND:
if self.verbose:# ['', MDP.W_WORKER, COMMAND, [OPTION], [MESSAGE]]
logging.info("I: sending %s to broker\n", command)
dump(msg)
self.creader.send_multipart(msg)# SEND THE MESSAGE TO THE SOCKET, FOR THE SBROKER TO PROCESS
def send(self, service, request):
"""Send request to broker and get reply by hook or crook.
Takes ownership of request message and destroys it when sent.
Returns the reply message or None if there was no reply.
"""
if not isinstance(request, list):
request = [request]
request = [MDP.C_CLIENT, service] + request
if self.verbose:
logging.warn("I: send request to '%s' service: ", service)
dump(request)
reply = None
retries = self.retries
while retries > 0:
self.client.send_multipart(request)
try:
items = self.poller.poll(self.timeout)
except KeyboardInterrupt:
break # interrupted
if items:
msg = self.client.recv_multipart()
if self.verbose:
logging.info("I: received reply:")
dump(msg)
# Don't try to handle errors, just assert noisily
assert len(msg) >= 3
header = msg.pop(0)
assert MDP.C_CLIENT == header
reply_service = msg.pop(0)
assert service == reply_service
reply = msg
break
else:
if retries:
logging.warn("W: no reply, reconnecting...")
self.reconnect_to_broker()
else:
logging.warn("W: permanent error, abandoning")
break
retries -= 1
return reply
def send(self, service, request):
"""Send request to broker
"""
if not isinstance(request, list):
request = [request]
# Prefix request with protocol frames
# Frame 0: empty (REQ emulation)
# Frame 1: "MDPCxy" (six bytes, MDP/Client x.y)
# Frame 2: Service name (printable string)
request = [b'', MDP.C_CLIENT, service] + request
if self.verbose:
logging.warn("I: send request to '%s' service: ", service)
dump(request)
self.client.send_multipart(request)
def process_worker(self, sender, msg):
"""Process message sent to us by a worker."""
assert len(msg) >= 1 # At least, command
command = msg.pop(0)
worker_ready = hexlify(sender) in self.workers
worker = self.require_worker(sender)
if (MDP.W_READY == command):
assert len(msg) >= 1 # At least, a service name
service = msg.pop(0)
# Not first command in session or Reserved service name
if (worker_ready
or service.startswith(self.INTERNAL_SERVICE_PREFIX)):
self.delete_worker(worker, True)
else:
# Attach worker to service and mark as idle
worker.service = self.require_service(service)
self.worker_waiting(worker)
elif (MDP.W_REPLY == command):
if (worker_ready):
# Remove & save client return envelope and insert the
# protocol header and service name, then rewrap envelope.
client = msg.pop(0)
empty = msg.pop(0)
assert empty == b''
msg = [client, b'', MDP.C_CLIENT, worker.service.name] + msg
self.socket.send_multipart(msg)
self.worker_waiting(worker)
else:
self.delete_worker(worker, True)
elif (MDP.W_HEARTBEAT == command):
if (worker_ready):
worker.expiry = time.time() + 1e-3 * self.HEARTBEAT_EXPIRY
else:
self.delete_worker(worker, True)
elif (MDP.W_DISCONNECT == command):
self.delete_worker(worker, False)
else:
logging.error("E: invalid message:")
dump(msg)
def send_to_broker(self, command, option=None, msg=None):
"""Send message to broker.
If no msg is provided, creates one internally
"""
if msg is None:
msg = []
elif not isinstance(msg, list):
msg = [msg]
if option:
msg = [option] + msg
msg = ['', MDP.W_WORKER, command] + msg
if self.verbose:
logging.info("I: sending %s to broker", command)
dump(msg)
self.worker.send_multipart(msg)
def send_to_creader(self, creader, command, option, msg=None, rtype=None):
if rtype is None:#CHECK IF THERE IS A VALID VALUE PASSED TO THE rype VARIABLE, IF NOT THEN
rtype = ['0']# ASSIGN 0 AS VALUE, MEANING NO REPLY NEEDED FROM THIS REQUEST, THIS IS USED FOR GETTING SESSIONS LIST !!!!
if msg is None:# CHECK IF THERE IS A VALID MESSAGE BEING PASSED TO THE msg VARIABLE, IF NOT THEN:
msg = []# TURN msg VARIABLE INTO A LIST
elif not isinstance(msg, list):# ELSE IF, msg IS NOT AN INSTANCE OF CLASS LIST THEN:
msg = [msg]# TURN MESSAGE msg VARIABLE INTO A LIST
#print "send_to_creader function received message: %s for: \n%s\n" % (msg,creader)#!!!!!!!!!!!!!!!
if option is not None:# CHECK IF OPTION VARIABLE HAS BEEN PASSED, IF NOT:
msg = [option] + msg# REWRAP THE msg INTO: [OPTION, MESSAGE]
msg = [creader.address, '', MDP.W_WORKER, command] + msg + rtype# REWRAP msg INTO: [CREADER.ADDRESS, '', CLASS, COMMAND, MESSAGE, REPLY(true/false)]
if self.verbose:# IF VERBOSE IS TRUE THEN:
logging.info("I: sending %r to creader", command)# LOG OUTPUT MESSAGE
dump(msg)# DUMP MESSAGE # print "A.SENT TO CREADER ADDRESS %s\nB.this message: %s\nC.reply was set:%s\n" % (creader, msg, rtype)
self.socket.send_multipart(msg)# SEND MESSAGE
def send_to_broker(self, command, option=None, msg=None):
"""Send message to broker.
If no msg is provided, creates one internally
"""
if msg is None:
msg = []
elif not isinstance(msg, list):
msg = [msg]
if option:
msg = [option] + msg
msg = [b'', MDP.W_WORKER, command] + msg
if self.verbose:
logging.info("I: sending %s to broker", command)
dump(msg)
self.worker.send_multipart(msg)
def process_worker(self, sender, msg):
"""Process message sent to us by a worker."""
assert len(msg) >= 1 # At least, command
command = msg.pop(0)
worker_ready = hexlify(sender) in self.workers
worker = self.require_worker(sender)
if (MDP.W_READY == command):
assert len(msg) >= 1 # At least, a service name
service = msg.pop(0)
# Not first command in session or Reserved service name
if (worker_ready or service.startswith(self.INTERNAL_SERVICE_PREFIX)):
self.delete_worker(worker, True)
else:
# Attach worker to service and mark as idle
worker.service = self.require_service(service)
self.worker_waiting(worker)
elif (MDP.W_REPLY == command):
if (worker_ready):
# Remove & save client return envelope and insert the
# protocol header and service name, then rewrap envelope.
client = msg.pop(0)
empty = msg.pop(0) # ?
msg = [client, '', MDP.C_CLIENT, worker.service.name] + msg
self.socket.send_multipart(msg)
self.worker_waiting(worker)
else:
self.delete_worker(worker, True)
elif (MDP.W_HEARTBEAT == command):
if (worker_ready):
worker.expiry = time.time() + 1e-3*self.HEARTBEAT_EXPIRY
else:
self.delete_worker(worker, True)
elif (MDP.W_DISCONNECT == command):
self.delete_worker(worker, False)
else:
logging.error("E: invalid message:")
dump(msg)
def send_to_worker(self, worker, command, option, msg=None):
"""Send message to worker.
If message is provided, sends that message.
"""
if msg is None:
msg = []
elif not isinstance(msg, list):
msg = [msg]
# Stack routing and protocol envelopes to start of message
# and routing envelope
if option is not None:
msg = [option] + msg
msg = [worker.address, '', MDP.W_WORKER, command] + msg
if self.verbose:
logging.info("I: sending %r to worker", command)
dump(msg)
self.socket.send_multipart(msg)
def recv(self):
try:
items = self.poller.poll(self.timeout)
except KeyboardInterrupt:
return
if items:
msg = self.client.recv_multipart()
if self.verbose:
logging.info("I: received reply:")
dump(msg)
assert len(msg) >= 4
empty = msg.pop(0)
header = msg.pop(0)
assert MDP.C_CLIENT == header
service = msg.pop(0)
return msg
else:
logging.warn("W: permanent error, abandoning request")
def send_to_broker(self, command, option=None, msg=None):
"""Send message to broker.
If no msg is provided, creates one internally
"""
if msg is None:
msg = []
elif not isinstance(msg, list):
msg = [msg]
if option:
msg = [
option.encode() if isinstance(option, str) else option
] + msg
msg = [b'', MDP.W_WORKER, command] + msg
if self.verbose:
logging.info(f"I: sending {MDP.format_msg(command)} to broker")
# use custom format function for printing messages
dump(msg, MDP.format_msg)
self.worker.send_multipart(msg)
def send_to_worker(self, worker, command, option, msg=None):
"""Send message to worker.
If message is provided, sends that message.
"""
if msg is None:
msg = []
elif not isinstance(msg, list):
msg = [msg]
# Stack routing and protocol envelopes to start of message
# and routing envelope
if option is not None:
msg = [option] + msg
msg = [worker.address, b'', MDP.W_WORKER, command] + msg
if self.verbose:
logging.info("I: sending %r to worker", command)
dump(msg)
self.socket.send_multipart(msg)
def uSession_mngr(self):
while True: # starting session loop
try:
items = self.poller.poll(self.HEARTBEAT_INTERVAL)
except KeyboardInterrupt:
print "Interrupted with keypress"
break# Interrupotion
if items:# check if something came in on the polling socket
msg = self.socket.recv_multipart()# assign that something to msg variable
if self.verbose:
logging.info("I: received message from connected Node")
dump(msg)
sender = msg.pop(0)# get the node that sent the message
empty = msg.pop(0)# empty delimiter frame of the message
assert empty == ''# check if message is formatted correctly
header = msg.pop(0) # get the node class that sent the message
if (NP.N_NODE == header):# check the node against node protocol
self.uNodeProcessing(sender, msg)# add node that sent the message to sessions
else:
logging.error("E: invalid message to sessions thread")
self.delete_dead_uNodes()
self.send_heartbeats_to_uNodes()
def handle_snapshot(self, msg):
"""snapshot requests"""
if len(msg) != 3 or msg[1] != b"ICANHAZ?":
print("E: bad request, aborting")
dump(msg)
self.loop.stop()
return
[identity, request, subtree] = msg
if subtree:
# Send state snapshot to client
route = Route(self.snapshot, identity, subtree)
# For each entry in kvmap, send kvmsg to client
for k, v in self.kvmap.items():
send_single(k, v, route)
# Now send END message with sequence number
logging.info(f"I: Sending state shapshot={self.sequence:d}")
self.snapshot.send(identity, zmq.SNDMORE)
kvmsg = KVMsg(self.sequence)
kvmsg.key = b"KTHXBAI"
kvmsg.body = subtree
kvmsg.send(self.snapshot)
def handle_snapshot(self, socket, msg):
"""snapshot requests"""
if msg[1] != "ICANHAZ?" or len(msg) != 3:
logging.error("E: bad request, aborting")
dump(msg)
self.bstar.loop.stop()
return
identity, request = msg[:2]
if len(msg) >= 3:
subtree = msg[2]
# Send state snapshot to client
route = Route(socket, identity, subtree)
# For each entry in kvmap, send kvmsg to client
for k,v in self.kvmap.items():
send_single(k,v,route)
# Now send END message with sequence number
logging.info("I: Sending state shapshot=%d" % self.sequence)
socket.send(identity, zmq.SNDMORE)
kvmsg = KVMsg(self.sequence)
kvmsg.key = "KTHXBAI"
kvmsg.body = subtree
kvmsg.send(socket)
def recv(self, reply=None):
"""Send reply, if any, to broker and wait for next request."""
# Format and send the reply if we were provided one
assert reply is not None or not self.expect_reply
if reply is not None:
assert self.reply_to is not None
reply = [self.reply_to, ''] + reply
self.send_to_broker(MDP.W_REPLY, msg=reply)
self.expect_reply = True
while True:
# Poll socket for a reply, with timeout
try:
items = self.poller.poll(self.timeout)
except KeyboardInterrupt:
break # Interrupted
if items:
msg = self.worker.recv_multipart()
if self.verbose:
logging.info("I: received message from broker: ")
dump(msg)
self.liveness = self.HEARTBEAT_LIVENESS
# Don't try to handle errors, just assert noisily
assert len(msg) >= 3
empty = msg.pop(0)
assert empty == ''
header = msg.pop(0)
assert header == MDP.W_WORKER
command = msg.pop(0)
if command == MDP.W_REQUEST:
# We should pop and save as many addresses as there are
# up to a null part, but for now, just save one...
self.reply_to = msg.pop(0)
# pop empty
assert msg.pop(0) == ''
return msg # We have a request to process
elif command == MDP.W_HEARTBEAT:
# Do nothing for heartbeats
pass
elif command == MDP.W_DISCONNECT:
self.reconnect_to_broker()
else :
logging.error("E: invalid input message: ")
dump(msg)
else:
self.liveness -= 1
if self.liveness == 0:
if self.verbose:
logging.warn("W: disconnected from broker - retrying...")
try:
time.sleep(1e-3*self.reconnect)
except KeyboardInterrupt:
break
self.reconnect_to_broker()
# Send HEARTBEAT if it's time
if time.time() > self.heartbeat_at:
self.send_to_broker(MDP.W_HEARTBEAT)
self.heartbeat_at = time.time() + 1e-3*self.heartbeat
logging.warn("W: interrupt received, killing worker...")
return None
def main():
logging.basicConfig(format="%(asctime)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO)
parser = ArgumentParser()
group = parser.add_mutually_exclusive_group()
group.add_argument("-p", "--primary", action="store_true", default=False)
group.add_argument("-b", "--backup", action="store_true", default=False)
parser.add_argument("-v", "--verbose", action="store_true", default=False)
args = parser.parse_args()
ctx = zmq.Context()
statepub = ctx.socket(zmq.PUB)
statesub = ctx.socket(zmq.SUB)
statesub.setsockopt_string(zmq.SUBSCRIBE, "")
frontend = ctx.socket(zmq.ROUTER)
fsm = BStarState(0, 0, 0)
if args.primary:
print("I: Primary master, waiting for backup (slave)")
frontend.bind("tcp://*:5001")
statepub.bind("tcp://*:5003")
statesub.connect("tcp://localhost:5004")
fsm.state = State.PRIMARY
elif args.backup:
print("I: Backup slave, waiting for primary (master)")
frontend.bind("tcp://*:5002")
statepub.bind("tcp://*:5004")
statesub.connect("tcp://localhost:5003")
statesub.setsockopt_string(zmq.SUBSCRIBE, "")
fsm.state = State.BACKUP
else:
print("Provide either -p{rimary} | -b{ackup}")
sys.exit(1)
send_state_at = int(time.time() * 1000 + HEARTBEAT)
poller = zmq.Poller()
poller.register(frontend, zmq.POLLIN)
poller.register(statesub, zmq.POLLIN)
while True:
time_left = send_state_at - int(time.time() * 1000)
if time_left < 0:
time_left = 0
socks = dict(poller.poll(time_left))
if socks.get(frontend) == zmq.POLLIN:
msg = frontend.recv_multipart()
if args.verbose:
logging.info('Received frontend.')
dump(msg)
fsm.event = Event.CLIENT_REQUEST
try:
run_fsm(fsm)
frontend.send_multipart(msg)
except BStarException:
del msg
if socks.get(statesub) == zmq.POLLIN:
msg = statesub.recv()
if args.verbose:
logging.info('Received state.')
dump(msg)
fsm.event = Event(int(msg))
del msg
try:
run_fsm(fsm)
fsm.peer_expiry = int(time.time() * 1000) + (2 * HEARTBEAT)
except BStarException:
break
else:
if not fsm.peer_expiry:
fsm.peer_expiry = int(time.time() * 1000) + PEERWAITLIMIT
if int(time.time() * 1000) >= fsm.peer_expiry:
fsm.event = Event.PEER_TIMEOUT
try:
run_fsm(fsm)
except BStarException:
pass
if int(time.time() * 1000) >= send_state_at:
if args.verbose:
logging.info(f'Sending state {fsm.state:d}.')
statepub.send(f"{fsm.state:d}".encode())
send_state_at = int(time.time() * 1000) + HEARTBEAT
# Cross-connected XREP sockets addressing each other
#
# Author: Lev Givon <lev(at)columbia(dot)edu>
import zmq
import time
import zhelpers
context = zmq.Context()
worker = context.socket(zmq.XREP)
worker.setsockopt(zmq.IDENTITY, "WORKER")
worker.bind("ipc://rtrouter.ipc")
server = context.socket(zmq.XREP)
server.setsockopt(zmq.IDENTITY, "SERVER")
server.connect("ipc://rtrouter.ipc")
time.sleep(1)
server.send_multipart(["WORKER", "", "send to worker"])
zhelpers.dump(worker)
worker.send_multipart(["SERVER", "", "send to server"])
zhelpers.dump(server)
# encoding: utf-8
#
# Demonstrate identities as used by the request-reply pattern. Run this
# program by itself.
#
# Author: Jeremy Avnet (brainsik) <spork(dash)zmq(at)theory(dot)org>
#
import zmq
import zhelpers
context = zmq.Context()
sink = context.socket(zmq.ROUTER)
sink.bind("inproc://example")
# First allow 0MQ to set the identity
anonymous = context.socket(zmq.DEALER)
anonymous.connect("inproc://example")
anonymous.send(b"ROUTER uses a generated 5 byte identity")
zhelpers.dump(sink)
# Then set the identity ourselves
identified = context.socket(zmq.DEALER)
identified.setsockopt(zmq.IDENTITY, b"PEER2")
identified.connect("inproc://example")
identified.send(b"ROUTER socket uses REQ's socket identity")
zhelpers.dump(sink)
import zhelpers
context = zmq.Context()
client = context.socket(zmq.XREP)
client.bind("ipc://routing.ipc")
worker = context.socket(zmq.REP)
worker.setsockopt(zmq.IDENTITY, "A")
worker.connect("ipc://routing.ipc")
# Wait for sockets to stabilize
time.sleep(1)
client.send("A", zmq.SNDMORE)
client.send("address 3", zmq.SNDMORE)
client.send("address 2", zmq.SNDMORE)
client.send("address 1", zmq.SNDMORE)
client.send("", zmq.SNDMORE)
client.send("This is the workload")
# Worker should get just the workload
zhelpers.dump(worker)
# We don't play with envelopes in the worker
worker.send("This is the reply")
# Now dump what we got off the XREP socket...
zhelpers.dump(client)
def main():
"""Load balancer main loop."""
# Prepare context and sockets
context = zmq.Context.instance()
frontend = context.socket(zmq.ROUTER)
frontend.bind("ipc://frontend.ipc")
backend = context.socket(zmq.ROUTER)
backend.bind("ipc://backend.ipc")
# Start background tasks
def start(task, *args):
process = multiprocessing.Process(target=task, args=args)
process.daemon = True
process.start()
for i in range(NBR_CLIENTS):
start(client_task, i)
for i in range(NBR_WORKERS):
start(worker_task, i)
# Initialize main loop state
count = NBR_CLIENTS
workers = []
poller = zmq.Poller()
# Only poll for requests from backend until workers are available
poller.register(backend, zmq.POLLIN)
while True:
sockets = dict(poller.poll())
if backend in sockets:
# Handle worker activity on the backend
request = backend.recv_multipart()
zhelpers.dump(request)
[worker, empty, client] = request[:3]
if not workers:
# Poll for clients now that a worker is available
poller.register(frontend, zmq.POLLIN)
workers.append(worker)
if client != b"READY" and len(request) > 3:
# If client reply, send rest back to frontend
[empty, reply] = request[3:]
frontend.send_multipart([client, b"", reply])
count -= 1
if not count:
break
if frontend in sockets:
# Get next client request, route to last-used worker
multipart = frontend.recv_multipart()
zhelpers.dump(multipart)
[client, empty, request] = multipart[:3]
worker = workers.pop(0)
backend.send_multipart([worker, b"", client, b"", request])
if not workers:
# Don't poll clients if no workers are available
poller.unregister(frontend)
# Clean up
backend.close()
frontend.close()
context.term()
import zmq
import zhelpers
context = zmq.Context()
client = context.socket(zmq.ROUTER)
client.bind("ipc://routing.ipc")
worker = context.socket(zmq.REP)
worker.setsockopt(zmq.IDENTITY, "A")
worker.connect("ipc://routing.ipc")
# Wait for sockets to stabilize
time.sleep(1)
client.send("A", zmq.SNDMORE)
client.send("address 3", zmq.SNDMORE)
client.send("address 2", zmq.SNDMORE)
client.send("address 1", zmq.SNDMORE)
client.send("", zmq.SNDMORE)
client.send("This is the workload")
# Worker should get just the workload
zhelpers.dump(worker)
# We don't play with envelopes in the worker
worker.send("This is the reply")
# Now dump what we got off the ROUTER socket...
zhelpers.dump(client)
def process_creader(self, sender, msg):
assert len(msg) >= 1# CHECK THAT MESSAGE HAS AT LEAST ONE ITEM, COMMAND
command = msg.pop(0)# ITEM.0 IN MSG IDENTIFIES THE COMMAND
creader_ready = hexlify(sender) in self.creaders# LOCATE SENDER IN CREADERS LIST
creader = self.require_creader(sender)# ASSIGN TO CREADER VARIABLE
if (MDP.W_READY == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_READY THEN:
assert len(msg) >= 1# ASSERT IF MESSAGE IS LONG ENOUGH TO PROCESS
cservice = msg.pop(0)# ITEM.0 IN MSG IDENTIFIES CSERVICE TO USE ON THAT MESSAGE
# Not first command in session or Reserved cservice name
if (creader_ready or cservice.startswith(self.INTERNAL_SERVICE_PREFIX)):# IF CREADER IS FOUND, OR RESERVED CSERVICE NAME THEN:
self.delete_creader(creader, True)# DELETE CREADER
else:# ELSE ATTACH CREADER TO CSERVICE AND MARK CREADER AS WAITING/IDLE
creader.service = self.require_cservice(cservice)# FIND CSERVICE FOR CREADER
self.creader_waiting(creader)# MARK CREADER AS WAITING/IDLE
elif (MDP.W_REPLY == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_REPLY THEN:
if (creader_ready):# IF CREADER IS MARKED AS READY print "got REPLY message, here is what it looks like: %s\n" % msg# !!!!!!!!!!!!!!!!
# Remove & save client return envelope and insert the
# protocol header and service name, then rewrap envelope.
client = msg.pop(0)# ITEM.0 IDENTIFIES CLIENT THAT SENT THE MESSAGE
msg.pop(0)#empty = msg.pop(0)# ITEM.1 IDENTIFIES EMPTY DELIMITER### ENABLE EMPTY LATER IF NEEDED
msg = [client, '', MDP.C_CLIENT, creader.service.name] + msg# REWRAP MSG ENVELOPE, IT BECOMES: [0.CLIENT,1.EMPTY,2.MDP HEADER,3.CSERVICEN,4.MESSAGE]
self.socket.send_multipart(msg)# SEND MESSAGE TO CREADER
self.creader_waiting(creader)# MARK CREADER AS WAITING AFTER SENDING MSG
else:# ELSE, CREADER IS NOT READY:
self.delete_creader(creader, True)# DELETE CREADER, BY PASSING CREADER TO CREADER_DELETE FUNCTION
elif (MDP.W_HEARTBEAT == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_HEARTBEAT THEN:
if (creader_ready):# IF CREADER IS MARKED AS READY
slist = dict.keys(self.cservices)# COLLECT ALL KEYS FROM CSERVICES DICTIONARY INTO A LIST
self.send_to_creader(creader, MDP.W_SESSIONS, None, slist, None)# USE A SEND COMMAND TO SEND THE LIST OF CSERVICES TO CREADER
creader.expiry = time.time() + 1e-3*self.HEARTBEAT_EXPIRY# RESET CREADER'S HEARTBEAT EXPIRY VALUE
else:# ELSE IF CREADER IS NOT READY:
self.delete_creader(creader, True)# DELETE CREADER, BY PASSING CREADER TO CREADER_DELETE FUNCTION
self.purge_creaders()# !!!!!!!!!!!!!!!!!!!!!!!!!!!
# elif (MDP.W_WEBRTC == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_WEBRTC THEN:
# if (creader_ready):
# a_cRD = creader
# b_cRD = self.find_creader_by_cservice(msg[0])
# fifoID = self.generate_fifo()
# if fifoID and a_cRD and b_cRD:
# fifoID = (str(self.WebRTC_SRV_addr) +":"+ str(self.WebRTC_SRV_port) +"/GET?"+ str(fifoID))
# self.send_to_creader(a_cRD, MDP.W_WEBRTC, None, fifoID, None)
# self.send_to_creader(b_cRD, MDP.W_WEBRTC, None, fifoID, None)
# else:
# pass
# else:
# pass
elif (MDP.W_POWERDOWN == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_POWERDOWN THEN:
if (creader_ready):
print "POWER DOWN RECEIVED BY SBROKER FOR:%s\n" % creader
pass
else:
print "POWER DOWN FAILURE"
pass
elif (MDP.W_DISCONNECT == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_DISCONNECT THEN:
self.purge_creaders()# !!!!!!!!!!!!!!!!!!!!!!!!!!!
self.delete_creader(creader, True)# DELETE CREADER, BY PASSING CREADER TO CREADER_DELETE FUNCTION
else:# ELSE, COMMAND IN THE MESSAGE FAILED MATCHING AGAINST THE MDP, MESSAGE WILL BE DUMPED AND STDOUT AS INVALID MESSAGE, BEACUSE IT DIDN'T MEET THE NECESSARY STRUCTURE.
logging.error("E: invalid message:")
dump(msg)
def recv(self, reply=None):
"""Send reply, if any, to broker and wait for next request."""
# Format and send the reply if we were provided one
assert reply is not None or not self.expect_reply
if reply is not None:
assert self.reply_to is not None
reply = [self.reply_to, b''] + reply
self.send_to_broker(MDP.W_REPLY, msg=reply)
self.expect_reply = True
while True:
# Poll socket for a reply, with timeout
try:
items = self.poller.poll(self.timeout)
except KeyboardInterrupt:
break # Interrupted
if items:
msg = self.worker.recv_multipart()
if self.verbose:
logging.info("I: received message from broker: ")
dump(msg)
self.liveness = self.HEARTBEAT_LIVENESS
# Don't try to handle errors, just assert noisily
assert len(msg) >= 3
empty = msg.pop(0)
assert empty == b''
header = msg.pop(0)
assert header == MDP.W_WORKER
command = msg.pop(0)
if command == MDP.W_REQUEST:
# We should pop and save as many addresses as there are
# up to a null part, but for now, just save one...
self.reply_to = msg.pop(0)
# pop empty
empty = msg.pop(0)
assert empty == b''
return msg # We have a request to process
elif command == MDP.W_HEARTBEAT:
# Do nothing for heartbeats
pass
elif command == MDP.W_DISCONNECT:
self.reconnect_to_broker()
else :
logging.error("E: invalid input message: ")
dump(msg)
else:
self.liveness -= 1
if self.liveness == 0:
if self.verbose:
logging.warn("W: disconnected from broker - retrying...")
try:
time.sleep(1e-3*self.reconnect)
except KeyboardInterrupt:
break
self.reconnect_to_broker()
# Send HEARTBEAT if it's time
if time.time() > self.heartbeat_at:
self.send_to_broker(MDP.W_HEARTBEAT)
self.heartbeat_at = time.time() + 1e-3*self.heartbeat
logging.warn("W: interrupt received, killing worker...")
return None
def recv(self, reply=None):#, status=True):
while self.recvState:# start a loop for receive function, this will listen to socket via POLLIN
try:# Poll socket for a reply, with timeout
items = self.poller.poll(self.timeout)# timeout assigned to POLLER on socket
except KeyboardInterrupt:# stop listening and looping if keyboard CTRL+C is pressed
break # Interrupted
if items and self.recvState:# CHECK IF MESSAGE CAME IN ON THE POLLER SOCKET, then:
msg = self.creader.recv_multipart()# assing incoming message to msg variable for processing
# if self.verbose:
# logging.info("I: received message from broker: \n")
# dump(msg)
self.liveness = self.HEARTBEAT_LIVENESS# Once message comes in, invoke HEARTBEAT FREQUENCY = 3
assert len(msg) >= 3# Don't try to handle errors, just assert noisily
empty = msg.pop(0)# strip the empty delimiter in the incoming message to empty variable
assert empty == ''# assert that emtpy delimeter is infact empty
header = msg.pop(0)# strip the header from the massage and assign in to the header variable
assert header == MDP.W_WORKER# assert that header is valid
command = msg.pop(0)# strip the command from the message and assign it to the command variable
if command == MDP.W_REQUEST:# check if command is a REQUEST, is so then:
print "request message arrived at CREADER here it is: %s\n" % msg
# We should pop and save as many addresses as there are
# up to a null part, but for now, just save one...
self.reply_to = msg.pop(0)# strip out the reply_to address from the message and assign it to variable
print "received MDP.REQUEST type message, going to reply with confirmation to: %s\n" % self.reply_to#!!!!!!!
rtype = msg.pop(len(msg)-1)# strip out the last item in the message: reply/no_reply 0/1 and assign it to the rtype variable !!!!!!!!!
assert rtype != ''# assert that rtype variable was passed with request !!!!!!!!!!!
if rtype != [0]:
print "reply has been requested for this request: rtype arg %s\n" % rtype# !!!!!!!!!!!!!
cWrep = [self.reply_to, '']+[msg[len(msg)-1]]# !!!!!!!!!!!!!!
print cWrep
self.send_to_sbroker(MDP.W_REPLY,msg=cWrep)# added
else:
print "message received by CREADER has rtype set to: %s\n" % rtype# !!!!!!!!!!!!!!!
print "no request needed for this message\n"# !!!!!!!!!!!!!!!
assert msg.pop(0) == ''# assert that the next part of the incoming message is an empty delimeter
self.cSessions = False
self.sessionFIFO = False
return msg # return actual message portion from the message, We now have a request to process
elif command == MDP.W_HEARTBEAT:# check if command is a HEARTBEAT, is so then:
# Do nothing for heartbeats
pass
elif command == MDP.W_SESSIONS:# check if command is a SESSIONS REQUEST, is so then:
self.cSessions = True
self.sessionFIFO = False
msg.pop(len(msg)-1)
self.sList = msg#['sList']+msg# PRINT OUT ACTIVE SESSIONS LIST, print msg
return self.cSessions
elif command == MDP.W_WEBRTC: # !!!!!!!!!!!!!!!!!!!!!!
self.sessionFIFO = True# !!!!!!!!!!!!!!!!!!!!!!
self.cSessions = False# !!!!!!!!!!!!!!!!!!!!!!
print "HOT DAMN!, GOT FIFO FROM THE BROKER"
msg.pop(len(msg)-1)# !!!!!!!!!!!!!!!!!!!!!!
print "THIS IS THE FIFO: %s\n" % msg
self.fList = msg# !!!!!!!!!!!!!!!!!!!!!!
print "turned to list: %s\n" % self.fList[(len(self.fList)-1)]
return self.sessionFIFO# !!!!!!!!!!!!!!!!!!!!!!
elif command == MDP.W_DISCONNECT:# check if command is a DISCONNECT, is so then:
# if self.powerDWN:
# print "powering down, disconnecting 000"
# self.disconnect_from_sbroker()
# break
# else:
self.reconnect_to_sbroker()# run the reconnect to sbroker command
else :# finally if message was not formed correctly by the CWRITER and SBROKER, let the user know
logging.error("E: invalid input message: ")
dump(msg)
else:# if items did not come in then
self.liveness -= 1# set liveness: decrease it by one
if self.liveness == 0:# if liveness is 0 then:
if self.verbose:# the user know that CREADER has dies and needs to re-connect to the SBROKER
logging.warn("W: disconnected from broker - retrying...")
try:# give it some time for the connection to die out
time.sleep(1e-3*self.reconnect)# sleep for a bit
except KeyboardInterrupt:# unless the script is interrupted by a CTRL+C command by suer
break
self.reconnect_to_sbroker()# try to reconnec to the SBROKER
# Send HEARTBEAT if it's time
if time.time() > self.heartbeat_at:
self.send_to_sbroker(MDP.W_HEARTBEAT)
self.heartbeat_at = time.time() + 1e-3*self.heartbeat
logging.warn("W: interrupt received, killing worker...")
self.cSessions = False
return None
