def process_bus_messages( self ): try: msg = self.lsub.recv() msgprefix, data = remap_utils.unpack_msg( msg ) if len(msgprefix) == 0: return True recipientid,msgtype,senderid = remap_utils.split_prefix(msgprefix) if msgtype[0] == '_': # node message self.process_core_message( msgtype, senderid, data ) elif msgtype == "corestatus": if senderid in self.cores: coredata = self.cores[ senderid ] coredata["ts_last_seen"] = time.time() coredata["progress"] = data["progress"] logger.info("Core %s progressed %d"%( senderid, coredata["progress"] )) self.forward_to_broker( msg ) elif msgtype == "complete": if senderid in self.cores: coredata = self.cores[ senderid ] logger.info("Core %s completed the job"%( senderid )) self.forward_to_broker( msg ) del self.cores[ senderid ] self.coresChanged = True else: # forward to broker instead self.forward_to_broker( msg ) return True except nn.NanoMsgAPIError as e: return False
def process_broker_messages(self): if self.bsub == None: # No broker is known yet. if self.brokerChanged: logger.info("The broker configuration changed.") self.setup_broker() if self.bsub == None: logger.info("Failed broker setup.") return False else: return False try: # Grab next msg from broker if any msg = self.bsub.recv() if msg != None and len(msg) > 0: msgprefix, data = remap_utils.unpack_msg(msg) recipientid, msgtype, senderid = remap_utils.split_prefix( msgprefix) if msgtype == "complete": self.update_corecomplete(recipientid, senderid, data) if msgtype == "corestatus": self.update_corestatus(recipientid, senderid, data) if msgtype == "raisehand": self.update_hands(recipientid, senderid, data) return True else: return False except nn.NanoMsgAPIError as e: return False
def process_broker_messages( self ): if self.bsub == None: # No broker is known yet. if self.brokerChanged: logger.info("The broker configuration changed.") self.setup_broker() if self.bsub == None: logger.info("Failed broker setup.") return False else: return False try: # Grab next msg from broker if any msg = self.bsub.recv() self.tot_m_rcv = self.tot_m_rcv + 1 if msg == None or len(msg)==0: return False msgprefix, data = remap_utils.unpack_msg( msg ) recipientid,msgtype,senderid = remap_utils.split_prefix(msgprefix) if msgtype == "showhands": self.handle_showhands( recipientid, senderid, data ) elif msgtype == "jobstart": #if recipientid == self.nodeid: self.handle_jobstart( recipientid, senderid, data ) else: # Forward to all cores for their processing. self.lpub.send(msg) return True except nn.NanoMsgAPIError as e: return False
def register( self ): self.set_node_timeout( 500 ) msgid = remap_utils.unique_id() logger.info( "Registering with node" ) self.forward( "node._hello.%d"%(self.pid), {"msgid":msgid,"pid":self.pid,"priority":self.priority} ) # The while loop will terminate as soon as node stops sending messages, # so this should be safe to do. while True: try: msg = self.sub.recv() msgprefix, data = remap_utils.unpack_msg( msg ) recipientid,msgtype,senderid = remap_utils.split_prefix(msgprefix) if msgtype != "_hey": continue replymsgid = remap_utils.safe_get(data, "msgid") if replymsgid == msgid: # this is us self.coreid = remap_utils.safe_get(data, "coreid" ) self.sub.set_string_option( nn.SUB, nn.SUB_UNSUBSCRIBE, "" ) self.sub.set_string_option( nn.SUB, nn.SUB_SUBSCRIBE, "global" ) self.sub.set_string_option( nn.SUB, nn.SUB_SUBSCRIBE, "local" ) self.sub.set_string_option( nn.SUB, nn.SUB_SUBSCRIBE, "notlocal" ) self.sub.set_string_option( nn.SUB, nn.SUB_SUBSCRIBE, self.coreid ) logger.info( "Received coreid %s."%( self.coreid )) return True except nn.NanoMsgAPIError as e: logger.error( "Node is currently not available." ) break logger.error( "Registration failed" ) return False
def process_broker_messages(self): if self.bsub == None: # No broker is known yet. if self.brokerChanged: logger.info("The broker configuration changed.") self.setup_broker() if self.bsub == None: logger.info("Failed broker setup.") return False else: return False try: # Grab next msg from broker if any msg = self.bsub.recv() self.tot_m_rcv = self.tot_m_rcv + 1 if msg == None or len(msg) == 0: return False msgprefix, data = remap_utils.unpack_msg(msg) recipientid, msgtype, senderid = remap_utils.split_prefix( msgprefix) if msgtype == "showhands": self.handle_showhands(recipientid, senderid, data) elif msgtype == "jobstart": #if recipientid == self.nodeid: self.handle_jobstart(recipientid, senderid, data) else: # Forward to all cores for their processing. self.lpub.send(msg) return True except nn.NanoMsgAPIError as e: return False
def process_bus_messages(self): try: msg = self.lsub.recv() msgprefix, data = remap_utils.unpack_msg(msg) if len(msgprefix) == 0: return True recipientid, msgtype, senderid = remap_utils.split_prefix( msgprefix) if msgtype[0] == '_': # node message self.process_core_message(msgtype, senderid, data) elif msgtype == "corestatus": if senderid in self.cores: coredata = self.cores[senderid] coredata["ts_last_seen"] = time.time() coredata["progress"] = data["progress"] logger.info("Core %s progressed %d" % (senderid, coredata["progress"])) self.forward_to_broker(msg) elif msgtype == "complete": if senderid in self.cores: coredata = self.cores[senderid] logger.info("Core %s completed the job" % (senderid)) self.forward_to_broker(msg) del self.cores[senderid] self.coresChanged = True else: # forward to broker instead self.forward_to_broker(msg) return True except nn.NanoMsgAPIError as e: return False
def process_broker_messages(self): if self.bsub == None: # No broker is known yet. if self.brokerChanged: logger.info("The broker configuration changed.") self.setup_broker() if self.bsub == None: logger.info("Failed broker setup.") return False else: return False try: # Grab next msg from broker if any msg = self.bsub.recv() if msg != None and len(msg) > 0: msgprefix, data = remap_utils.unpack_msg(msg) recipientid, msgtype, senderid = remap_utils.split_prefix(msgprefix) if msgtype == "complete": self.update_corecomplete(recipientid, senderid, data) if msgtype == "corestatus": self.update_corestatus(recipientid, senderid, data) if msgtype == "raisehand": self.update_hands(recipientid, senderid, data) return True else: return False except nn.NanoMsgAPIError as e: return False
def process_node_messages( self ): try: msg = self.sub.recv() msgprefix, data = remap_utils.unpack_msg( msg ) recipientid,msgtype,senderid = remap_utils.split_prefix(msgprefix) if recipientid == self.coreid: # This is directed at this core specifically, so it's more of a req/rep type self.process_personal_message( msgtype, senderid, data ) elif recipientid == "global": self.process_global_message( msgtype, senderid, data ) elif recipientid == "local": self.process_local_message( msgtype, senderid, data ) elif recipientid == "notlocal": self.process_global_message( msgtype, senderid, data ) elif recipientid == "node": self.process_node_message( msgtype, senderid, data ) else: logger.info("Unrecognized message type %s, sent by %s"%( msgtype, senderid ) ) return True except nn.NanoMsgAPIError as e: return False
def work( self ): if not self.ready: return True surveyormsg = None try: surveyormsg = remap_utils.decode( self.surveyor.recv() ) except nn.NanoMsgAPIError as e: return True if surveyormsg[0] == 'S': # Shift messages if self.mode != MODE_MSGS: self.mode = MODE_MSGS # We haven't done this in a previous step. Due to recovery, it might be # used by the initiator to get others up to speed. for key, (vertex,messages,messagesNext) in self.vertices.items(): self.vertices[ key ] = ( vertex, messagesNext, [] ) self.surveyor.send( "D" ) return True if surveyormsg[0] == 'H': self.mode = MODE_HALT logger.info("Halting core.") self.surveyor.close() return False if surveyormsg[0] == 'P': if self.mode != MODE_PROCESS: # First time in this state, we need to grab all messages and # allocate them to vertex queue self.mode = MODE_PROCESS logger.info("Processing messages 1") while True: try: msg = self.vsub.recv() prefix, data = remap_utils.unpack_msg( msg ) if prefix in self.vertices: # This vertex is indeed on this host. Add the message to its new msg list for next iteration vertex, messages, messagesNext = self.vertices[ prefix ] messagesNext.append( data ) except nn.NanoMsgAPIError as e: logger.error( "No more messages available." ) break else: logger.info("Processing messages 2") # doing things twice does not make a difference. Second time around, just throw away all messages while True: try: msg = self.vsub.recv() print("Received and thrown away: ", msg) except nn.NanoMsgAPIError as e: logger.error( "No more messages available." ) break self.surveyor.send( "D" ) return True self.mode = MODE_RUN self.superstep = int(surveyormsg) mainHalt = True for key, (vertex,messages,messagesNext) in self.vertices.items(): vertex, halt = self.app.compute( self.forward, self.subscribe, self.unsubscribe, self.superstep, vertex, messages ) if vertex != None: # Store the new vertex object in its place, maintaining the messagesNext list as we know it self.vertices[ key ] = (vertex,[],messagesNext) if not halt: mainHalt = False if mainHalt: self.surveyor.send( "H" ) else: self.surveyor.send( "D" ) return True