示例#1
0
文件: client_api.py 项目: wox88/bolt
	def recv(self):
		"""
		Returns the reply message or None if there was no reply.
		"""
		try:
			items = self.poller.poll(self.timeout)
		except KeyboardInterrupt:
			return # interrupted

		if items:
			# if we got a reply, process it
			msg = self.client.recv_multipart()
			if self.verbose:
				logging.info("I: received reply:")
				dump(msg)

			# Don't try to handle errors, just assert noisily
			assert len(msg) >= 4

			empty = msg.pop(0)
			header = msg.pop(0)
			assert MDP.C_CLIENT == header

			service = msg.pop(0)
			return msg
		else:      
			logging.warn("W: permanent error, abandoning request")
示例#2
0
    def handle_snapshot(self, msg):
        """snapshot requests"""

        logging.info("I: received state request on snapshot: %s" % msg)
        if len(msg) != 3 or msg[1] != "ICANHAZ?":
            print "E: bad request, aborting"
            dump(msg)
            self.loop.stop()
            return
        identity, request, subtree = msg
        if subtree:
            # Send state snapshot to client
            route = Route(self.snapshot, identity, subtree)

            # For each entry in kvmap, send kvmsg to client
            for k, v in self.kvmap.items():
                logging.info("I: Sending snapshot: %s" % v)
                send_single(k, v, route)

            # Now send END message with sequence number
            self.snapshot.send(identity, zmq.SNDMORE)
            kvmsg = KVMsg(self.sequence)
            kvmsg.key = "KTHXBAI"
            kvmsg.body = subtree
            logging.info("I: Sending snapshot end message: %s" % kvmsg)
            kvmsg.send(self.snapshot)
示例#3
0
    def mediate(self):
        """Main broker work happens here"""
        while True:
            try:
                items = self.poller.poll(self.HEARTBEAT_INTERVAL)
            except KeyboardInterrupt:
                break # Interrupted
            if items:
                msg = self.socket.recv_multipart()
                if self.verbose:
                    logging.info("I: received message:")
                    dump(msg)

                sender = msg.pop(0)
                empty = msg.pop(0)
                assert empty == ''
                header = msg.pop(0)

                if (MDP.C_CLIENT == header):
                    self.process_client(sender, msg)
                elif (MDP.W_WORKER == header):
                    self.process_worker(sender, msg)
                else:
                    logging.error("E: invalid message:")
                    dump(msg)

            self.purge_workers()
            self.send_heartbeats()
示例#4
0
    def mediate(self):#Main SBROKER work happens here, TEST ALL INCOMING MESSAGES FOR VALIDITY
        while True:# LOOP
            try:
                items = self.poller.poll(self.HEARTBEAT_INTERVAL)# SET POLLING INTERVAL ON SOCKET
            except KeyboardInterrupt:#STOP LOOP IF CTRL+C IS PRESSED
                break# Interrupted
            if items:# IF DATA CAME IN ON POLLING SOCKET
                msg = self.socket.recv_multipart()# ASSIGN INCOMING MESSAGE TO 'MSG' LIST VARIABLE
                if self.verbose:
                    logging.info("I: received message:")
                    dump(msg)

                sender = msg.pop(0)# ITEM.0 IN MSG IDENTIFIES THE CWRITER
                empty = msg.pop(0)# ITEM.1 IN MSG IDENTIFIES EMPTY DELIMITER
                assert empty == ''# CHECK IF EMPTY DELIMITER IS ACTUALLY EMPTY
                header = msg.pop(0)# ITEM.2 IN MSG IDENTIFIES CWRITER THAT SENT THE MESSAGE
                if (MDP.C_CLIENT == header):# CHECK MDP PROTOCOL AGAINST CWRITER TO CHECK THAT CWRITER LINKED TO SOCKET
                    self.process_cwriter(sender, msg)# START PROCESSING THE CWRITER, RUN PROCESSING FUNCTION ON IT
                elif (MDP.W_WORKER == header):# CHECK MDP PROTOCOL AGAINST CREADER TO CHECK THAT CREADER LINKED TO SOCKET
                    self.process_creader(sender, msg)# START PROCESSING THE CWRITER, RUN PROCESSING FUNCTION ON IT
                else:# INVALIDATE MESSAGE IF IT DIN'T COME FROM A CREADER OR A CWRITER, OR WAS INVALIDLY CONSTRUCTED
                    logging.error("E: invalid message:")
                    dump(msg)
            self.purge_creaders()# RUN FUNCTION TO CHECK FOR REMOVED/DISCONNECTED CREADERS AND DELETE THEM AND THEIR SESSIONS
            self.send_heartbeats()# RUN FUNCTION FOR SENDING HEARTBEATS TO CREADERS THAT ARE IN ACTIVE SESSIONS LIST
示例#5
0
def main():
    verbose = '-v' in sys.argv
    
    ctx = zmq.Context()
    # Prepare server socket with predictable identity
    bind_endpoint = "tcp://*:5555"
    connect_endpoint = "tcp://localhost:5555"
    server = ctx.socket(zmq.ROUTER)
    server.identity = connect_endpoint
    server.bind(bind_endpoint)
    print "I: service is ready at", bind_endpoint
    
    while True:
        try:
            request = server.recv_multipart()
        except:
            break # Interrupted
        # Frame 0: identity of client
        # Frame 1: PING, or client control frame
        # Frame 2: request body
        address, control = request[:2]
        reply = [address, control]
        if control == "PING":
            reply[1] = "PONG"
        else:
            reply.append("OK")
        if verbose:
            dump(reply)
        server.send_multipart(reply)
    print "W: interrupted"
示例#6
0
def main():
    verbose = '-v' in sys.argv

    ctx = zmq.Context()
    # Prepare server socket with predictable identity
    bind_endpoint = "tcp://*:5555"
    connect_endpoint = "tcp://localhost:5555"
    server = ctx.socket(zmq.ROUTER)
    server.identity = connect_endpoint
    server.bind(bind_endpoint)
    print "I: service is ready at", bind_endpoint

    while True:
        try:
            request = server.recv_multipart()
        except:
            break  # Interrupted
        # Frame 0: identity of client
        # Frame 1: PING, or client control frame
        # Frame 2: request body
        address, control = request[:2]
        reply = [address, control]
        if control == "PING":
            reply[1] = "PONG"
        else:
            reply.append("OK")
        if verbose:
            dump(reply)
        server.send_multipart(reply)
    print "W: interrupted"
示例#7
0
    def mediate(self):
        """Main broker work happens here"""
        while True:
            try:
                items = self.poller.poll(self.HEARTBEAT_INTERVAL)
            except KeyboardInterrupt:
                break  # Interrupted
            if items:
                msg = self.socket.recv_multipart()
                if self.verbose:
                    logging.info("I: received message:")
                    dump(msg)

                sender = msg.pop(0)
                empty = msg.pop(0)
                assert empty == b''
                header = msg.pop(0)

                if (MDP.C_CLIENT == header):
                    self.process_client(sender, msg)
                elif (MDP.W_WORKER == header):
                    self.process_worker(sender, msg)
                else:
                    logging.error("E: invalid message:")
                    dump(msg)

            self.purge_workers()
            self.send_heartbeats()
示例#8
0
    def recv(self):
        """Returns the reply message or None if there was no reply."""
        try:
            items = self.poller.poll(self.timeout)
        except KeyboardInterrupt:
            return # interrupted

        if items:
            # if we got a reply, process it
            msg = self.client.recv_multipart()
            if self.verbose:
                logging.info("I: received reply:")
                dump(msg)

            # Don't try to handle errors, just assert noisily
            assert len(msg) >= 4

            empty = msg.pop(0)
            header = msg.pop(0)
            assert MDP.C_CLIENT == header

            service = msg.pop(0)
            return msg
        else:
            logging.warn("W: permanent error, abandoning request")
示例#9
0
def listener_thread(pipe):
    # Print everything that arrives on pipe
    while True:
        try:
            dump(pipe.recv_multipart())
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                break  # Interrupted
示例#10
0
    def send(self, service, request):
        if not isinstance(request, list):
            request = [request]

        # Prefix request with protocol frames
        # Frame 0: empty (REQ emulation)
        # Frame 1: "MDPCxy" (six bytes, MDP/Client x.y)
        # Frame 2: Service name (printable string)
        
        request = ['', MDP.C_CLIENT, service] + request
        if self.verbose:
            logging.warn("I: send request to '%s' service: ", service)
            dump(request)
        self.client.send_multipart(request)
示例#11
0
	def send_to_sbroker(self, command, option=None, msg=None):# FUNCTION IS CALLED FOR SENDING MESSAGES TO SBROKER
		if msg is None:# CHECK IF MESSAGE IS PROVIED TO THE FUNCTION
			msg = []# IF NOT THEN SET MESSAGE TO EMPTY LIST
		elif not isinstance(msg, list):# CHECK THAT MESSAGE THAT HAS BEEN PROVIED IS TYPE LIST
			msg = [msg]# IF NOT THEN MAKE PROVIED MESSAGE INTO A LIST

		if option:# IF OPTION IS PROVIED THEN REWRAP THE MESSAGE BY PLACING OPTION FIRST:
			msg = [option] + msg# NOW THE MESSAGE LOOKS LIKE THIS: [[OPTION],[MESSAGE]]

		msg = ['', MDP.W_WORKER, command] + msg# REWRAP THE MESSAGE AGAIN WITH EMPTY DELIMITER, CREADER ID AND COMMAND:
		if self.verbose:# ['', MDP.W_WORKER, COMMAND, [OPTION], [MESSAGE]]
			logging.info("I: sending %s to broker\n", command)
			dump(msg)
		self.creader.send_multipart(msg)# SEND THE MESSAGE TO THE SOCKET, FOR THE SBROKER TO PROCESS
示例#12
0
    def send(self, service, request):
        """Send request to broker and get reply by hook or crook.

        Takes ownership of request message and destroys it when sent.
        Returns the reply message or None if there was no reply.
        """
        if not isinstance(request, list):
            request = [request]
        request = [MDP.C_CLIENT, service] + request
        if self.verbose:
            logging.warn("I: send request to '%s' service: ", service)
            dump(request)
        reply = None

        retries = self.retries
        while retries > 0:
            self.client.send_multipart(request)
            try:
                items = self.poller.poll(self.timeout)
            except KeyboardInterrupt:
                break # interrupted

            if items:
                msg = self.client.recv_multipart()
                if self.verbose:
                    logging.info("I: received reply:")
                    dump(msg)

                # Don't try to handle errors, just assert noisily
                assert len(msg) >= 3

                header = msg.pop(0)
                assert MDP.C_CLIENT == header

                reply_service = msg.pop(0)
                assert service == reply_service

                reply = msg
                break
            else:
                if retries:
                    logging.warn("W: no reply, reconnecting...")
                    self.reconnect_to_broker()
                else:
                    logging.warn("W: permanent error, abandoning")
                    break
                retries -= 1

        return reply
示例#13
0
    def send(self, service, request):
        """Send request to broker
        """
        if not isinstance(request, list):
            request = [request]

        # Prefix request with protocol frames
        # Frame 0: empty (REQ emulation)
        # Frame 1: "MDPCxy" (six bytes, MDP/Client x.y)
        # Frame 2: Service name (printable string)

        request = [b'', MDP.C_CLIENT, service] + request
        if self.verbose:
            logging.warn("I: send request to '%s' service: ", service)
            dump(request)
        self.client.send_multipart(request)
示例#14
0
    def process_worker(self, sender, msg):
        """Process message sent to us by a worker."""
        assert len(msg) >= 1  # At least, command

        command = msg.pop(0)

        worker_ready = hexlify(sender) in self.workers

        worker = self.require_worker(sender)

        if (MDP.W_READY == command):
            assert len(msg) >= 1  # At least, a service name
            service = msg.pop(0)
            # Not first command in session or Reserved service name
            if (worker_ready
                    or service.startswith(self.INTERNAL_SERVICE_PREFIX)):
                self.delete_worker(worker, True)
            else:
                # Attach worker to service and mark as idle
                worker.service = self.require_service(service)
                self.worker_waiting(worker)

        elif (MDP.W_REPLY == command):
            if (worker_ready):
                # Remove & save client return envelope and insert the
                # protocol header and service name, then rewrap envelope.
                client = msg.pop(0)
                empty = msg.pop(0)
                assert empty == b''
                msg = [client, b'', MDP.C_CLIENT, worker.service.name] + msg
                self.socket.send_multipart(msg)
                self.worker_waiting(worker)
            else:
                self.delete_worker(worker, True)

        elif (MDP.W_HEARTBEAT == command):
            if (worker_ready):
                worker.expiry = time.time() + 1e-3 * self.HEARTBEAT_EXPIRY
            else:
                self.delete_worker(worker, True)

        elif (MDP.W_DISCONNECT == command):
            self.delete_worker(worker, False)
        else:
            logging.error("E: invalid message:")
            dump(msg)
示例#15
0
    def send_to_broker(self, command, option=None, msg=None):
        """Send message to broker.
        If no msg is provided, creates one internally
        """
        if msg is None:
            msg = []
        elif not isinstance(msg, list):
            msg = [msg]

        if option:
            msg = [option] + msg

        msg = ['', MDP.W_WORKER, command] + msg
        if self.verbose:
            logging.info("I: sending %s to broker", command)
            dump(msg)
        self.worker.send_multipart(msg)
示例#16
0
    def send_to_creader(self, creader, command, option, msg=None, rtype=None):
        if rtype is None:#CHECK IF THERE IS A VALID VALUE PASSED TO THE rype VARIABLE, IF NOT THEN
            rtype = ['0']# ASSIGN 0 AS VALUE, MEANING NO REPLY NEEDED FROM THIS REQUEST, THIS IS USED FOR GETTING SESSIONS LIST !!!!
        if msg is None:# CHECK IF THERE IS A VALID MESSAGE BEING PASSED TO THE msg VARIABLE, IF NOT THEN:
            msg = []# TURN msg VARIABLE INTO A LIST
        elif not isinstance(msg, list):# ELSE IF, msg IS NOT AN INSTANCE OF CLASS LIST THEN:
            msg = [msg]# TURN MESSAGE msg VARIABLE INTO A LIST
        #print "send_to_creader function received message: %s for: \n%s\n" % (msg,creader)#!!!!!!!!!!!!!!!
        if option is not None:# CHECK IF OPTION VARIABLE HAS BEEN PASSED, IF NOT:
            msg = [option] + msg# REWRAP THE msg INTO: [OPTION, MESSAGE]
        msg = [creader.address, '', MDP.W_WORKER, command] + msg + rtype# REWRAP msg INTO: [CREADER.ADDRESS, '', CLASS, COMMAND, MESSAGE, REPLY(true/false)]

        if self.verbose:# IF VERBOSE IS TRUE THEN:
            logging.info("I: sending %r to creader", command)# LOG OUTPUT MESSAGE
            dump(msg)# DUMP MESSAGE # print "A.SENT TO CREADER ADDRESS %s\nB.this message: %s\nC.reply was set:%s\n" % (creader, msg, rtype)
        
        self.socket.send_multipart(msg)# SEND MESSAGE
示例#17
0
    def send_to_broker(self, command, option=None, msg=None):
        """Send message to broker.

        If no msg is provided, creates one internally
        """
        if msg is None:
            msg = []
        elif not isinstance(msg, list):
            msg = [msg]

        if option:
            msg = [option] + msg

        msg = [b'', MDP.W_WORKER, command] + msg
        if self.verbose:
            logging.info("I: sending %s to broker", command)
            dump(msg)
        self.worker.send_multipart(msg)
示例#18
0
    def process_worker(self, sender, msg):
        """Process message sent to us by a worker."""
        assert len(msg) >= 1 # At least, command

        command = msg.pop(0)

        worker_ready = hexlify(sender) in self.workers

        worker = self.require_worker(sender)

        if (MDP.W_READY == command):
            assert len(msg) >= 1 # At least, a service name
            service = msg.pop(0)
            # Not first command in session or Reserved service name
            if (worker_ready or service.startswith(self.INTERNAL_SERVICE_PREFIX)):
                self.delete_worker(worker, True)
            else:
                # Attach worker to service and mark as idle
                worker.service = self.require_service(service)
                self.worker_waiting(worker)

        elif (MDP.W_REPLY == command):
            if (worker_ready):
                # Remove & save client return envelope and insert the
                # protocol header and service name, then rewrap envelope.
                client = msg.pop(0)
                empty = msg.pop(0) # ?
                msg = [client, '', MDP.C_CLIENT, worker.service.name] + msg
                self.socket.send_multipart(msg)
                self.worker_waiting(worker)
            else:
                self.delete_worker(worker, True)

        elif (MDP.W_HEARTBEAT == command):
            if (worker_ready):
                worker.expiry = time.time() + 1e-3*self.HEARTBEAT_EXPIRY
            else:
                self.delete_worker(worker, True)

        elif (MDP.W_DISCONNECT == command):
            self.delete_worker(worker, False)
        else:
            logging.error("E: invalid message:")
            dump(msg)
示例#19
0
    def send_to_worker(self, worker, command, option, msg=None):
        """Send message to worker.
        If message is provided, sends that message.
        """

        if msg is None:
            msg = []
        elif not isinstance(msg, list):
            msg = [msg]

        # Stack routing and protocol envelopes to start of message
        # and routing envelope
        if option is not None:
            msg = [option] + msg
        msg = [worker.address, '', MDP.W_WORKER, command] + msg

        if self.verbose:
            logging.info("I: sending %r to worker", command)
            dump(msg)

        self.socket.send_multipart(msg)
示例#20
0
    def recv(self):
        try:
            items = self.poller.poll(self.timeout)
        except KeyboardInterrupt:
            return

        if items:
            msg = self.client.recv_multipart()
            if self.verbose:
                logging.info("I: received reply:")
                dump(msg)

            assert len(msg) >= 4

            empty = msg.pop(0)
            header = msg.pop(0)
            assert MDP.C_CLIENT == header

            service = msg.pop(0)
            return msg
        else:
            logging.warn("W: permanent error, abandoning request")
示例#21
0
    def send_to_broker(self, command, option=None, msg=None):
        """Send message to broker.

        If no msg is provided, creates one internally
        """
        if msg is None:
            msg = []
        elif not isinstance(msg, list):
            msg = [msg]

        if option:
            msg = [
                option.encode() if isinstance(option, str) else option
            ] + msg

        msg = [b'', MDP.W_WORKER, command] + msg
        if self.verbose:
            logging.info(f"I: sending {MDP.format_msg(command)} to broker")
            # use custom format function for printing messages
            dump(msg, MDP.format_msg)

        self.worker.send_multipart(msg)
示例#22
0
    def send_to_worker(self, worker, command, option, msg=None):
        """Send message to worker.

        If message is provided, sends that message.
        """

        if msg is None:
            msg = []
        elif not isinstance(msg, list):
            msg = [msg]

        # Stack routing and protocol envelopes to start of message
        # and routing envelope
        if option is not None:
            msg = [option] + msg
        msg = [worker.address, b'', MDP.W_WORKER, command] + msg

        if self.verbose:
            logging.info("I: sending %r to worker", command)
            dump(msg)

        self.socket.send_multipart(msg)
示例#23
0
		def uSession_mngr(self):
			while True: # starting session loop
				try:
					items = self.poller.poll(self.HEARTBEAT_INTERVAL)
				except KeyboardInterrupt:
					print "Interrupted with keypress"
					break# Interrupotion
				if items:# check if something came in on the polling socket
					msg = self.socket.recv_multipart()# assign that something to msg variable
					if self.verbose:
						logging.info("I: received message from connected Node")
						dump(msg)

					sender = msg.pop(0)# get the node that sent the message
					empty = msg.pop(0)#  empty delimiter frame of the message
					assert empty == ''# check if message is formatted correctly
					header = msg.pop(0) # get the node class that sent the message
					if (NP.N_NODE == header):# check the node against node protocol
						self.uNodeProcessing(sender, msg)# add node that sent the message to sessions
					else:
						logging.error("E: invalid message to sessions thread")
					self.delete_dead_uNodes()
					self.send_heartbeats_to_uNodes()
示例#24
0
    def handle_snapshot(self, msg):
        """snapshot requests"""
        if len(msg) != 3 or msg[1] != b"ICANHAZ?":
            print("E: bad request, aborting")
            dump(msg)
            self.loop.stop()
            return
        [identity, request, subtree] = msg
        if subtree:
            # Send state snapshot to client
            route = Route(self.snapshot, identity, subtree)

            # For each entry in kvmap, send kvmsg to client
            for k, v in self.kvmap.items():
                send_single(k, v, route)

            # Now send END message with sequence number
            logging.info(f"I: Sending state shapshot={self.sequence:d}")
            self.snapshot.send(identity, zmq.SNDMORE)
            kvmsg = KVMsg(self.sequence)
            kvmsg.key = b"KTHXBAI"
            kvmsg.body = subtree
            kvmsg.send(self.snapshot)
示例#25
0
    def handle_snapshot(self, socket, msg):
        """snapshot requests"""
        if msg[1] != "ICANHAZ?" or len(msg) != 3:
            logging.error("E: bad request, aborting")
            dump(msg)
            self.bstar.loop.stop()
            return
        identity, request = msg[:2]
        if len(msg) >= 3:
            subtree = msg[2]
            # Send state snapshot to client
            route = Route(socket, identity, subtree)

            # For each entry in kvmap, send kvmsg to client
            for k,v in self.kvmap.items():
                send_single(k,v,route)
            
            # Now send END message with sequence number
            logging.info("I: Sending state shapshot=%d" % self.sequence)
            socket.send(identity, zmq.SNDMORE)
            kvmsg = KVMsg(self.sequence)
            kvmsg.key = "KTHXBAI"
            kvmsg.body = subtree
            kvmsg.send(socket)
示例#26
0
    def recv(self, reply=None):
        """Send reply, if any, to broker and wait for next request."""
        # Format and send the reply if we were provided one
        assert reply is not None or not self.expect_reply

        if reply is not None:
            assert self.reply_to is not None
            reply = [self.reply_to, ''] + reply
            self.send_to_broker(MDP.W_REPLY, msg=reply)
        
        self.expect_reply = True
        
        while True:
            # Poll socket for a reply, with timeout
            try:
                items = self.poller.poll(self.timeout)
            except KeyboardInterrupt:
                break # Interrupted
            
            if items:
                msg = self.worker.recv_multipart()
                if self.verbose:
                    logging.info("I: received message from broker: ")
                    dump(msg)
                
                self.liveness = self.HEARTBEAT_LIVENESS
                # Don't try to handle errors, just assert noisily
                assert len(msg) >= 3

                empty = msg.pop(0)
                assert empty == ''

                header = msg.pop(0)
                assert header == MDP.W_WORKER

                command = msg.pop(0)
                if command == MDP.W_REQUEST:
                    # We should pop and save as many addresses as there are
                    # up to a null part, but for now, just save one...
                    self.reply_to = msg.pop(0)
                    # pop empty
                    assert msg.pop(0) == ''
                    
                    return msg # We have a request to process
                elif command == MDP.W_HEARTBEAT:
                    # Do nothing for heartbeats
                    pass
                elif command == MDP.W_DISCONNECT:
                    self.reconnect_to_broker()
                else :
                    logging.error("E: invalid input message: ")
                    dump(msg)
            
            else:
                self.liveness -= 1
                if self.liveness == 0:
                    if self.verbose:
                        logging.warn("W: disconnected from broker - retrying...")
                    try:
                        time.sleep(1e-3*self.reconnect)
                    except KeyboardInterrupt:
                        break
                    self.reconnect_to_broker()
            
            # Send HEARTBEAT if it's time
            if time.time() > self.heartbeat_at:
                self.send_to_broker(MDP.W_HEARTBEAT)
                self.heartbeat_at = time.time() + 1e-3*self.heartbeat
        
        logging.warn("W: interrupt received, killing worker...")
        return None
示例#27
0
def main():
    logging.basicConfig(format="%(asctime)s %(message)s",
                        datefmt="%Y-%m-%d %H:%M:%S",
                        level=logging.INFO)

    parser = ArgumentParser()
    group = parser.add_mutually_exclusive_group()
    group.add_argument("-p", "--primary", action="store_true", default=False)
    group.add_argument("-b", "--backup", action="store_true", default=False)
    parser.add_argument("-v", "--verbose", action="store_true", default=False)
    args = parser.parse_args()

    ctx = zmq.Context()
    statepub = ctx.socket(zmq.PUB)
    statesub = ctx.socket(zmq.SUB)
    statesub.setsockopt_string(zmq.SUBSCRIBE, "")
    frontend = ctx.socket(zmq.ROUTER)

    fsm = BStarState(0, 0, 0)

    if args.primary:
        print("I: Primary master, waiting for backup (slave)")
        frontend.bind("tcp://*:5001")
        statepub.bind("tcp://*:5003")
        statesub.connect("tcp://localhost:5004")
        fsm.state = State.PRIMARY
    elif args.backup:
        print("I: Backup slave, waiting for primary (master)")
        frontend.bind("tcp://*:5002")
        statepub.bind("tcp://*:5004")
        statesub.connect("tcp://localhost:5003")
        statesub.setsockopt_string(zmq.SUBSCRIBE, "")
        fsm.state = State.BACKUP
    else:
        print("Provide either -p{rimary} | -b{ackup}")
        sys.exit(1)

    send_state_at = int(time.time() * 1000 + HEARTBEAT)
    poller = zmq.Poller()
    poller.register(frontend, zmq.POLLIN)
    poller.register(statesub, zmq.POLLIN)

    while True:
        time_left = send_state_at - int(time.time() * 1000)
        if time_left < 0:
            time_left = 0
        socks = dict(poller.poll(time_left))
        if socks.get(frontend) == zmq.POLLIN:
            msg = frontend.recv_multipart()
            if args.verbose:
                logging.info('Received frontend.')
                dump(msg)
            fsm.event = Event.CLIENT_REQUEST
            try:
                run_fsm(fsm)
                frontend.send_multipart(msg)
            except BStarException:
                del msg

        if socks.get(statesub) == zmq.POLLIN:
            msg = statesub.recv()
            if args.verbose:
                logging.info('Received state.')
                dump(msg)
            fsm.event = Event(int(msg))
            del msg
            try:
                run_fsm(fsm)
                fsm.peer_expiry = int(time.time() * 1000) + (2 * HEARTBEAT)
            except BStarException:
                break
        else:
            if not fsm.peer_expiry:
                fsm.peer_expiry = int(time.time() * 1000) + PEERWAITLIMIT
            if int(time.time() * 1000) >= fsm.peer_expiry:
                fsm.event = Event.PEER_TIMEOUT
                try:
                    run_fsm(fsm)
                except BStarException:
                    pass

        if int(time.time() * 1000) >= send_state_at:
            if args.verbose:
                logging.info(f'Sending state {fsm.state:d}.')
            statepub.send(f"{fsm.state:d}".encode())
            send_state_at = int(time.time() * 1000) + HEARTBEAT
示例#28
0
# Cross-connected XREP sockets addressing each other
#
# Author: Lev Givon <lev(at)columbia(dot)edu>

import zmq
import time
import zhelpers

context = zmq.Context()

worker = context.socket(zmq.XREP)
worker.setsockopt(zmq.IDENTITY, "WORKER")
worker.bind("ipc://rtrouter.ipc")

server = context.socket(zmq.XREP)
server.setsockopt(zmq.IDENTITY, "SERVER")
server.connect("ipc://rtrouter.ipc")

time.sleep(1)

server.send_multipart(["WORKER", "", "send to worker"])
zhelpers.dump(worker)

worker.send_multipart(["SERVER", "", "send to server"])
zhelpers.dump(server)
示例#29
0
# encoding: utf-8
#
#   Demonstrate identities as used by the request-reply pattern.  Run this
#   program by itself.
#
#   Author: Jeremy Avnet (brainsik) <spork(dash)zmq(at)theory(dot)org>
#

import zmq
import zhelpers

context = zmq.Context()

sink = context.socket(zmq.ROUTER)
sink.bind("inproc://example")

# First allow 0MQ to set the identity
anonymous = context.socket(zmq.DEALER)
anonymous.connect("inproc://example")
anonymous.send(b"ROUTER uses a generated 5 byte identity")
zhelpers.dump(sink)

# Then set the identity ourselves
identified = context.socket(zmq.DEALER)
identified.setsockopt(zmq.IDENTITY, b"PEER2")
identified.connect("inproc://example")
identified.send(b"ROUTER socket uses REQ's socket identity")
zhelpers.dump(sink)
示例#30
0
import zhelpers


context = zmq.Context()
client = context.socket(zmq.XREP)
client.bind("ipc://routing.ipc")

worker = context.socket(zmq.REP)
worker.setsockopt(zmq.IDENTITY, "A")
worker.connect("ipc://routing.ipc")

# Wait for sockets to stabilize
time.sleep(1)

client.send("A", zmq.SNDMORE)
client.send("address 3", zmq.SNDMORE)
client.send("address 2", zmq.SNDMORE)
client.send("address 1", zmq.SNDMORE)
client.send("", zmq.SNDMORE)
client.send("This is the workload")

# Worker should get just the workload
zhelpers.dump(worker)

# We don't play with envelopes in the worker
worker.send("This is the reply")

# Now dump what we got off the XREP socket...
zhelpers.dump(client)
示例#31
0
def main():
    """Load balancer main loop."""
    # Prepare context and sockets
    context = zmq.Context.instance()
    frontend = context.socket(zmq.ROUTER)
    frontend.bind("ipc://frontend.ipc")
    backend = context.socket(zmq.ROUTER)
    backend.bind("ipc://backend.ipc")

    # Start background tasks
    def start(task, *args):
        process = multiprocessing.Process(target=task, args=args)
        process.daemon = True
        process.start()

    for i in range(NBR_CLIENTS):
        start(client_task, i)
    for i in range(NBR_WORKERS):
        start(worker_task, i)

    # Initialize main loop state
    count = NBR_CLIENTS
    workers = []
    poller = zmq.Poller()
    # Only poll for requests from backend until workers are available
    poller.register(backend, zmq.POLLIN)

    while True:
        sockets = dict(poller.poll())

        if backend in sockets:
            # Handle worker activity on the backend
            request = backend.recv_multipart()
            zhelpers.dump(request)
            [worker, empty, client] = request[:3]
            if not workers:
                # Poll for clients now that a worker is available
                poller.register(frontend, zmq.POLLIN)
            workers.append(worker)
            if client != b"READY" and len(request) > 3:
                # If client reply, send rest back to frontend
                [empty, reply] = request[3:]
                frontend.send_multipart([client, b"", reply])
                count -= 1
                if not count:
                    break

        if frontend in sockets:
            # Get next client request, route to last-used worker
            multipart = frontend.recv_multipart()
            zhelpers.dump(multipart)
            [client, empty, request] = multipart[:3]
            worker = workers.pop(0)
            backend.send_multipart([worker, b"", client, b"", request])
            if not workers:
                # Don't poll clients if no workers are available
                poller.unregister(frontend)

    # Clean up
    backend.close()
    frontend.close()
    context.term()
示例#32
0
import zmq

import zhelpers

context = zmq.Context()
client = context.socket(zmq.ROUTER)
client.bind("ipc://routing.ipc")

worker = context.socket(zmq.REP)
worker.setsockopt(zmq.IDENTITY, "A")
worker.connect("ipc://routing.ipc")

# Wait for sockets to stabilize
time.sleep(1)

client.send("A", zmq.SNDMORE)
client.send("address 3", zmq.SNDMORE)
client.send("address 2", zmq.SNDMORE)
client.send("address 1", zmq.SNDMORE)
client.send("", zmq.SNDMORE)
client.send("This is the workload")

# Worker should get just the workload
zhelpers.dump(worker)

# We don't play with envelopes in the worker
worker.send("This is the reply")

# Now dump what we got off the ROUTER socket...
zhelpers.dump(client)
示例#33
0
    def process_creader(self, sender, msg):
        assert len(msg) >= 1# CHECK THAT MESSAGE HAS AT LEAST ONE ITEM, COMMAND
        command = msg.pop(0)# ITEM.0 IN MSG IDENTIFIES THE COMMAND
        creader_ready = hexlify(sender) in self.creaders# LOCATE SENDER IN CREADERS LIST
        creader = self.require_creader(sender)# ASSIGN TO CREADER VARIABLE
        if (MDP.W_READY == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_READY THEN:
            assert len(msg) >= 1# ASSERT IF MESSAGE IS LONG ENOUGH TO PROCESS
            cservice = msg.pop(0)# ITEM.0 IN MSG IDENTIFIES CSERVICE TO USE ON THAT MESSAGE
            # Not first command in session or Reserved cservice name
            if (creader_ready or cservice.startswith(self.INTERNAL_SERVICE_PREFIX)):# IF CREADER IS FOUND, OR RESERVED CSERVICE NAME THEN:

                self.delete_creader(creader, True)# DELETE CREADER
            else:# ELSE ATTACH CREADER TO CSERVICE AND MARK CREADER AS WAITING/IDLE
                creader.service = self.require_cservice(cservice)# FIND CSERVICE FOR CREADER
                self.creader_waiting(creader)# MARK CREADER AS WAITING/IDLE

        elif (MDP.W_REPLY == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_REPLY THEN:
            if (creader_ready):# IF CREADER IS MARKED AS READY print "got REPLY message, here is what it looks like: %s\n" % msg# !!!!!!!!!!!!!!!!
                # Remove & save client return envelope and insert the
                # protocol header and service name, then rewrap envelope.
                client = msg.pop(0)# ITEM.0 IDENTIFIES CLIENT THAT SENT THE MESSAGE
                msg.pop(0)#empty = msg.pop(0)# ITEM.1 IDENTIFIES EMPTY DELIMITER### ENABLE EMPTY LATER IF NEEDED 
                msg = [client, '', MDP.C_CLIENT, creader.service.name] + msg# REWRAP MSG ENVELOPE, IT BECOMES: [0.CLIENT,1.EMPTY,2.MDP HEADER,3.CSERVICEN,4.MESSAGE]
                self.socket.send_multipart(msg)# SEND MESSAGE TO CREADER
                self.creader_waiting(creader)# MARK CREADER AS WAITING AFTER SENDING MSG
            else:# ELSE, CREADER IS NOT READY:
                self.delete_creader(creader, True)# DELETE CREADER, BY PASSING CREADER TO CREADER_DELETE FUNCTION

        elif (MDP.W_HEARTBEAT == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_HEARTBEAT THEN:
            if (creader_ready):# IF CREADER IS MARKED AS READY
                slist = dict.keys(self.cservices)# COLLECT ALL KEYS FROM CSERVICES DICTIONARY INTO A LIST           
                self.send_to_creader(creader, MDP.W_SESSIONS, None, slist, None)# USE A SEND COMMAND TO SEND THE LIST OF CSERVICES TO CREADER
                creader.expiry = time.time() + 1e-3*self.HEARTBEAT_EXPIRY# RESET CREADER'S HEARTBEAT EXPIRY VALUE
            else:# ELSE IF CREADER IS NOT READY:
                self.delete_creader(creader, True)# DELETE CREADER, BY PASSING CREADER TO CREADER_DELETE FUNCTION
                self.purge_creaders()# !!!!!!!!!!!!!!!!!!!!!!!!!!!


#        elif (MDP.W_WEBRTC == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_WEBRTC THEN:
#            if (creader_ready):
#                a_cRD = creader
#                b_cRD = self.find_creader_by_cservice(msg[0])
#                fifoID = self.generate_fifo()
#                if fifoID and a_cRD and b_cRD:
#                    fifoID = (str(self.WebRTC_SRV_addr) +":"+ str(self.WebRTC_SRV_port) +"/GET?"+ str(fifoID))
#                    self.send_to_creader(a_cRD, MDP.W_WEBRTC, None, fifoID, None)
#                    self.send_to_creader(b_cRD, MDP.W_WEBRTC, None, fifoID, None)
#                else:
#                    pass
#            else:
#                pass

        elif (MDP.W_POWERDOWN == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_POWERDOWN THEN:
            if (creader_ready):
                print "POWER DOWN RECEIVED BY SBROKER FOR:%s\n" % creader
                pass
            else:
                print "POWER DOWN FAILURE"
                pass

        elif (MDP.W_DISCONNECT == command):# CHECK INCOMING MESSAGE COMMAND AGAINST MDP IF IT'S W_DISCONNECT THEN:
            self.purge_creaders()# !!!!!!!!!!!!!!!!!!!!!!!!!!!
            self.delete_creader(creader, True)# DELETE CREADER, BY PASSING CREADER TO CREADER_DELETE FUNCTION

        else:# ELSE, COMMAND IN THE MESSAGE FAILED MATCHING AGAINST THE MDP, MESSAGE WILL BE DUMPED AND STDOUT AS INVALID MESSAGE, BEACUSE IT DIDN'T MEET THE NECESSARY STRUCTURE.
            logging.error("E: invalid message:")
            dump(msg)            
示例#34
0
    def recv(self, reply=None):
        """Send reply, if any, to broker and wait for next request."""
        # Format and send the reply if we were provided one
        assert reply is not None or not self.expect_reply

        if reply is not None:
            assert self.reply_to is not None
            reply = [self.reply_to, b''] + reply
            self.send_to_broker(MDP.W_REPLY, msg=reply)

        self.expect_reply = True

        while True:
            # Poll socket for a reply, with timeout
            try:
                items = self.poller.poll(self.timeout)
            except KeyboardInterrupt:
                break # Interrupted

            if items:
                msg = self.worker.recv_multipart()
                if self.verbose:
                    logging.info("I: received message from broker: ")
                    dump(msg)

                self.liveness = self.HEARTBEAT_LIVENESS
                # Don't try to handle errors, just assert noisily
                assert len(msg) >= 3

                empty = msg.pop(0)
                assert empty == b''

                header = msg.pop(0)
                assert header == MDP.W_WORKER

                command = msg.pop(0)
                if command == MDP.W_REQUEST:
                    # We should pop and save as many addresses as there are
                    # up to a null part, but for now, just save one...
                    self.reply_to = msg.pop(0)
                    # pop empty
                    empty = msg.pop(0)
                    assert empty == b''

                    return msg # We have a request to process
                elif command == MDP.W_HEARTBEAT:
                    # Do nothing for heartbeats
                    pass
                elif command == MDP.W_DISCONNECT:
                    self.reconnect_to_broker()
                else :
                    logging.error("E: invalid input message: ")
                    dump(msg)

            else:
                self.liveness -= 1
                if self.liveness == 0:
                    if self.verbose:
                        logging.warn("W: disconnected from broker - retrying...")
                    try:
                        time.sleep(1e-3*self.reconnect)
                    except KeyboardInterrupt:
                        break
                    self.reconnect_to_broker()

            # Send HEARTBEAT if it's time
            if time.time() > self.heartbeat_at:
                self.send_to_broker(MDP.W_HEARTBEAT)
                self.heartbeat_at = time.time() + 1e-3*self.heartbeat

        logging.warn("W: interrupt received, killing worker...")
        return None
示例#35
0
	def recv(self, reply=None):#, status=True):
		while self.recvState:# start a loop for receive function, this will listen to socket via POLLIN 
			try:# Poll socket for a reply, with timeout
				items = self.poller.poll(self.timeout)# timeout assigned to POLLER on socket
			except KeyboardInterrupt:# stop listening and looping if keyboard CTRL+C is pressed
				break # Interrupted

			if items and self.recvState:# CHECK IF MESSAGE CAME IN ON THE POLLER SOCKET, then:
				msg = self.creader.recv_multipart()# assing incoming message to msg variable for processing
				# if self.verbose:
				# 	logging.info("I: received message from broker: \n")
				# 	dump(msg)
				self.liveness = self.HEARTBEAT_LIVENESS# Once message comes in, invoke HEARTBEAT FREQUENCY = 3
				assert len(msg) >= 3# Don't try to handle errors, just assert noisily
				empty = msg.pop(0)# strip the empty delimiter in the incoming message to empty variable
				assert empty == ''# assert that emtpy delimeter is infact empty
				header = msg.pop(0)# strip the header from the massage and assign in to the header variable
				assert header == MDP.W_WORKER# assert that header is valid
				command = msg.pop(0)# strip the command from the message and assign it to the command variable
				if command == MDP.W_REQUEST:# check if command is a REQUEST, is so then:
					print "request message arrived at CREADER here it is: %s\n" % msg
					# We should pop and save as many addresses as there are
					# up to a null part, but for now, just save one...
					self.reply_to = msg.pop(0)# strip out the reply_to address from the message and assign it to variable
					print "received MDP.REQUEST type message, going to reply with confirmation to: %s\n" % self.reply_to#!!!!!!!
					rtype = msg.pop(len(msg)-1)# strip out the last item in the message: reply/no_reply 0/1 and assign it to the rtype variable !!!!!!!!!
					assert rtype != ''# assert that rtype variable was passed with request !!!!!!!!!!!
					if rtype != [0]:
						print "reply has been requested for this request: rtype arg %s\n" % rtype# !!!!!!!!!!!!!
						cWrep = [self.reply_to, '']+[msg[len(msg)-1]]# !!!!!!!!!!!!!!
						print cWrep
						self.send_to_sbroker(MDP.W_REPLY,msg=cWrep)# added 
					else:
						print "message received by CREADER has rtype set to: %s\n" % rtype# !!!!!!!!!!!!!!!
						print "no request needed for this message\n"# !!!!!!!!!!!!!!!
					assert msg.pop(0) == ''# assert that the next part of the incoming message is an empty delimeter
					self.cSessions = False
					self.sessionFIFO = False
					return msg # return actual message portion from the message, We now have a request to process
				elif command == MDP.W_HEARTBEAT:# check if command is a HEARTBEAT, is so then:
					# Do nothing for heartbeats
					pass
				elif command == MDP.W_SESSIONS:# check if command is a SESSIONS REQUEST, is so then:
					self.cSessions = True
					self.sessionFIFO = False
					msg.pop(len(msg)-1)
					self.sList = msg#['sList']+msg# PRINT OUT ACTIVE SESSIONS LIST, print msg
					return self.cSessions
				elif command == MDP.W_WEBRTC: # !!!!!!!!!!!!!!!!!!!!!!
					self.sessionFIFO = True# !!!!!!!!!!!!!!!!!!!!!!
					self.cSessions = False# !!!!!!!!!!!!!!!!!!!!!!
					print "HOT DAMN!, GOT FIFO FROM THE BROKER"
					msg.pop(len(msg)-1)# !!!!!!!!!!!!!!!!!!!!!!
					print "THIS IS THE FIFO: %s\n" % msg
					self.fList = msg# !!!!!!!!!!!!!!!!!!!!!!
					print "turned to list: %s\n" % self.fList[(len(self.fList)-1)]
					return self.sessionFIFO# !!!!!!!!!!!!!!!!!!!!!!
				elif command == MDP.W_DISCONNECT:# check if command is a DISCONNECT, is so then:
					# if self.powerDWN:
					# 	print "powering down, disconnecting 000"
					# 	self.disconnect_from_sbroker()
					# 	break
					# else:
						self.reconnect_to_sbroker()# run the reconnect to sbroker command
				else :# finally if message was not formed correctly by the CWRITER and SBROKER, let the user know
					logging.error("E: invalid input message: ")
					dump(msg)

			else:# if items did not come in then
				self.liveness -= 1# set liveness: decrease it by one
				if self.liveness == 0:# if liveness is 0 then:
					if self.verbose:# the user know that CREADER has dies and needs to re-connect to the SBROKER
						logging.warn("W: disconnected from broker - retrying...")
					try:# give it some time for the connection to die out
						time.sleep(1e-3*self.reconnect)# sleep for a bit
					except KeyboardInterrupt:# unless the script is interrupted by a CTRL+C command by suer
						break
					self.reconnect_to_sbroker()# try to reconnec to the SBROKER

			# Send HEARTBEAT if it's time
			if time.time() > self.heartbeat_at:
				self.send_to_sbroker(MDP.W_HEARTBEAT)
				self.heartbeat_at = time.time() + 1e-3*self.heartbeat

		logging.warn("W: interrupt received, killing worker...")
		self.cSessions = False
		return None