def process_bus_messages( self ):
try:
msg = self.lsub.recv()
msgprefix, data = remap_utils.unpack_msg( msg )
if len(msgprefix) == 0:
return True
recipientid,msgtype,senderid = remap_utils.split_prefix(msgprefix)
if msgtype[0] == '_':
# node message
self.process_core_message( msgtype, senderid, data )
elif msgtype == "corestatus":
if senderid in self.cores:
coredata = self.cores[ senderid ]
coredata["ts_last_seen"] = time.time()
coredata["progress"] = data["progress"]
logger.info("Core %s progressed %d"%( senderid, coredata["progress"] ))
self.forward_to_broker( msg )
elif msgtype == "complete":
if senderid in self.cores:
coredata = self.cores[ senderid ]
logger.info("Core %s completed the job"%( senderid ))
self.forward_to_broker( msg )
del self.cores[ senderid ]
self.coresChanged = True
else:
# forward to broker instead
self.forward_to_broker( msg )
return True
except nn.NanoMsgAPIError as e:
return False
def process_broker_messages(self):
if self.bsub == None:
# No broker is known yet.
if self.brokerChanged:
logger.info("The broker configuration changed.")
self.setup_broker()
if self.bsub == None:
logger.info("Failed broker setup.")
return False
else:
return False
try:
# Grab next msg from broker if any
msg = self.bsub.recv()
if msg != None and len(msg) > 0:
msgprefix, data = remap_utils.unpack_msg(msg)
recipientid, msgtype, senderid = remap_utils.split_prefix(
msgprefix)
if msgtype == "complete":
self.update_corecomplete(recipientid, senderid, data)
if msgtype == "corestatus":
self.update_corestatus(recipientid, senderid, data)
if msgtype == "raisehand":
self.update_hands(recipientid, senderid, data)
return True
else:
return False
except nn.NanoMsgAPIError as e:
return False
def process_broker_messages( self ):
if self.bsub == None:
# No broker is known yet.
if self.brokerChanged:
logger.info("The broker configuration changed.")
self.setup_broker()
if self.bsub == None:
logger.info("Failed broker setup.")
return False
else:
return False
try:
# Grab next msg from broker if any
msg = self.bsub.recv()
self.tot_m_rcv = self.tot_m_rcv + 1
if msg == None or len(msg)==0:
return False
msgprefix, data = remap_utils.unpack_msg( msg )
recipientid,msgtype,senderid = remap_utils.split_prefix(msgprefix)
if msgtype == "showhands":
self.handle_showhands( recipientid, senderid, data )
elif msgtype == "jobstart":
#if recipientid == self.nodeid:
self.handle_jobstart( recipientid, senderid, data )
else:
# Forward to all cores for their processing.
self.lpub.send(msg)
return True
except nn.NanoMsgAPIError as e:
return False
def register( self ):
self.set_node_timeout( 500 )
msgid = remap_utils.unique_id()
logger.info( "Registering with node" )
self.forward( "node._hello.%d"%(self.pid), {"msgid":msgid,"pid":self.pid,"priority":self.priority} )
# The while loop will terminate as soon as node stops sending messages,
# so this should be safe to do.
while True:
try:
msg = self.sub.recv()
msgprefix, data = remap_utils.unpack_msg( msg )
recipientid,msgtype,senderid = remap_utils.split_prefix(msgprefix)
if msgtype != "_hey":
continue
replymsgid = remap_utils.safe_get(data, "msgid")
if replymsgid == msgid:
# this is us
self.coreid = remap_utils.safe_get(data, "coreid" )
self.sub.set_string_option( nn.SUB, nn.SUB_UNSUBSCRIBE, "" )
self.sub.set_string_option( nn.SUB, nn.SUB_SUBSCRIBE, "global" )
self.sub.set_string_option( nn.SUB, nn.SUB_SUBSCRIBE, "local" )
self.sub.set_string_option( nn.SUB, nn.SUB_SUBSCRIBE, "notlocal" )
self.sub.set_string_option( nn.SUB, nn.SUB_SUBSCRIBE, self.coreid )
logger.info( "Received coreid %s."%( self.coreid ))
return True
except nn.NanoMsgAPIError as e:
logger.error( "Node is currently not available." )
break
logger.error( "Registration failed" )
return False
def process_broker_messages(self):
if self.bsub == None:
# No broker is known yet.
if self.brokerChanged:
logger.info("The broker configuration changed.")
self.setup_broker()
if self.bsub == None:
logger.info("Failed broker setup.")
return False
else:
return False
try:
# Grab next msg from broker if any
msg = self.bsub.recv()
self.tot_m_rcv = self.tot_m_rcv + 1
if msg == None or len(msg) == 0:
return False
msgprefix, data = remap_utils.unpack_msg(msg)
recipientid, msgtype, senderid = remap_utils.split_prefix(
msgprefix)
if msgtype == "showhands":
self.handle_showhands(recipientid, senderid, data)
elif msgtype == "jobstart":
#if recipientid == self.nodeid:
self.handle_jobstart(recipientid, senderid, data)
else:
# Forward to all cores for their processing.
self.lpub.send(msg)
return True
except nn.NanoMsgAPIError as e:
return False
def process_bus_messages(self):
try:
msg = self.lsub.recv()
msgprefix, data = remap_utils.unpack_msg(msg)
if len(msgprefix) == 0:
return True
recipientid, msgtype, senderid = remap_utils.split_prefix(
msgprefix)
if msgtype[0] == '_':
# node message
self.process_core_message(msgtype, senderid, data)
elif msgtype == "corestatus":
if senderid in self.cores:
coredata = self.cores[senderid]
coredata["ts_last_seen"] = time.time()
coredata["progress"] = data["progress"]
logger.info("Core %s progressed %d" %
(senderid, coredata["progress"]))
self.forward_to_broker(msg)
elif msgtype == "complete":
if senderid in self.cores:
coredata = self.cores[senderid]
logger.info("Core %s completed the job" % (senderid))
self.forward_to_broker(msg)
del self.cores[senderid]
self.coresChanged = True
else:
# forward to broker instead
self.forward_to_broker(msg)
return True
except nn.NanoMsgAPIError as e:
return False
def process_broker_messages(self):
if self.bsub == None:
# No broker is known yet.
if self.brokerChanged:
logger.info("The broker configuration changed.")
self.setup_broker()
if self.bsub == None:
logger.info("Failed broker setup.")
return False
else:
return False
try:
# Grab next msg from broker if any
msg = self.bsub.recv()
if msg != None and len(msg) > 0:
msgprefix, data = remap_utils.unpack_msg(msg)
recipientid, msgtype, senderid = remap_utils.split_prefix(msgprefix)
if msgtype == "complete":
self.update_corecomplete(recipientid, senderid, data)
if msgtype == "corestatus":
self.update_corestatus(recipientid, senderid, data)
if msgtype == "raisehand":
self.update_hands(recipientid, senderid, data)
return True
else:
return False
except nn.NanoMsgAPIError as e:
return False
def process_node_messages( self ):
try:
msg = self.sub.recv()
msgprefix, data = remap_utils.unpack_msg( msg )
recipientid,msgtype,senderid = remap_utils.split_prefix(msgprefix)
if recipientid == self.coreid:
# This is directed at this core specifically, so it's more of a req/rep type
self.process_personal_message( msgtype, senderid, data )
elif recipientid == "global":
self.process_global_message( msgtype, senderid, data )
elif recipientid == "local":
self.process_local_message( msgtype, senderid, data )
elif recipientid == "notlocal":
self.process_global_message( msgtype, senderid, data )
elif recipientid == "node":
self.process_node_message( msgtype, senderid, data )
else:
logger.info("Unrecognized message type %s, sent by %s"%( msgtype, senderid ) )
return True
except nn.NanoMsgAPIError as e:
return False
def work( self ):
if not self.ready:
return True
surveyormsg = None
try:
surveyormsg = remap_utils.decode( self.surveyor.recv() )
except nn.NanoMsgAPIError as e:
return True
if surveyormsg[0] == 'S':
# Shift messages
if self.mode != MODE_MSGS:
self.mode = MODE_MSGS
# We haven't done this in a previous step. Due to recovery, it might be
# used by the initiator to get others up to speed.
for key, (vertex,messages,messagesNext) in self.vertices.items():
self.vertices[ key ] = ( vertex, messagesNext, [] )
self.surveyor.send( "D" )
return True
if surveyormsg[0] == 'H':
self.mode = MODE_HALT
logger.info("Halting core.")
self.surveyor.close()
return False
if surveyormsg[0] == 'P':
if self.mode != MODE_PROCESS:
# First time in this state, we need to grab all messages and
# allocate them to vertex queue
self.mode = MODE_PROCESS
logger.info("Processing messages 1")
while True:
try:
msg = self.vsub.recv()
prefix, data = remap_utils.unpack_msg( msg )
if prefix in self.vertices:
# This vertex is indeed on this host. Add the message to its new msg list for next iteration
vertex, messages, messagesNext = self.vertices[ prefix ]
messagesNext.append( data )
except nn.NanoMsgAPIError as e:
logger.error( "No more messages available." )
break
else:
logger.info("Processing messages 2")
# doing things twice does not make a difference. Second time around, just throw away all messages
while True:
try:
msg = self.vsub.recv()
print("Received and thrown away: ", msg)
except nn.NanoMsgAPIError as e:
logger.error( "No more messages available." )
break
self.surveyor.send( "D" )
return True
self.mode = MODE_RUN
self.superstep = int(surveyormsg)
mainHalt = True
for key, (vertex,messages,messagesNext) in self.vertices.items():
vertex, halt = self.app.compute( self.forward, self.subscribe, self.unsubscribe, self.superstep, vertex, messages )
if vertex != None:
# Store the new vertex object in its place, maintaining the messagesNext list as we know it
self.vertices[ key ] = (vertex,[],messagesNext)
if not halt:
mainHalt = False
if mainHalt:
self.surveyor.send( "H" )
else:
self.surveyor.send( "D" )
return True