def send_ncAcks(self, filled_outgoingBfr_ids):
try:
for inst_id in filled_outgoingBfr_ids:
logger.debug("Length of in_to_out bfr:"+str(self.outgoing_ncAckBfr[inst_id].qsize()))
item = self.outgoing_ncAckBfr[inst_id].get()
encoded_msg = self.gen_msg(item)
encoded_msg = encoded_msg + asynchat_msg_terminator
# reply_id indicates the seq_no of the msg whose ACK is being sent
# extract the session_id and subseq_no from reply_id
session_id = self.convert_to_int(item.reply_id[:self.seq_no_partition_size])
subseq_no = self.convert_to_int(item.reply_id[self.seq_no_partition_size:])
for msg_id in self.unprocessed_gnSeqNos[inst_id]:
if msg_id[0] == session_id and msg_id[1] == subseq_no:
self.unprocessed_gnSeqNos[inst_id].remove(msg_id)
# Check whether this is not a response for the old msg with old session_id
#if session_id == self.gn_session_id[inst_id]:
self.sent_ncAckBfr[inst_id].append((\
session_id, subseq_no, encoded_msg))
# as its an old msg don't send it
#else:
#logger.critical("Session id of GN: "+str(inst_id)+\
#" has changed so discarding msg.............\n\n")
#send_msg = 0
#if send_msg:
self.send_msg_to_gn(inst_id, encoded_msg)
self.outgoing_ncAckBfr[inst_id].task_done()
except Exception as inst:
logger.critical("Exception in send_ncAcks: " + str(inst))
def run(self):
try:
# initializes data structures for cloud
self.init_nc_related_node_data_structures('cloud')
logger.debug("Starting " + self.thread_name)
wait_time_set=0
while True:
filled_outgoingBfr_ids=self.get_filled_outgoing_bfr_ids()
while (not self.incoming_moduleMsgBfr.empty()) or (not self.outgoing_ncMsgBfr['cloud'].empty()) or filled_outgoingBfr_ids:
self.process_incoming_moduleMsg()
if not self.outgoing_ncMsgBfr['cloud'].empty():
item = self.outgoing_ncMsgBfr['cloud'].get()
encoded_msg = self.gen_msg(item)
self.send_msg_to_cloud(encoded_msg)
self.send_ncAcks(filled_outgoingBfr_ids)
filled_outgoingBfr_ids=[]
time.sleep(0.0001)
wait_time_set=0
else:
if wait_time_set==0:
wait_time = time.time() + wait_time_for_next_msg
wait_time_set=1;
if wait_time > time.time():
time.sleep(0.0001)
else:
time.sleep(0.1)
except Exception as inst:
logger.critical("Exception in gn_msgs_bufr_mngr run: " + str(inst))
def __init__(self, thread_name):
try:
threading.Thread.__init__(self)
# can be used by logging module for printing messages related to this thread
self.thread_name = "Thread_" + thread_name
self.daemon = True
self.log_file_name = "session_ids"
#self.handler_vector_table = {}
#self.initialize_handler_vector_table()
###############################
"""
# Below are dictionaries representing different types of buffers
# for each GN
# key=inst_id
"""
self.outgoing_ncMsgBfr = {}
self.sent_ncMsgBfr = {} # for future use
self.outgoing_ncAckBfr = {}
# Stores only sent replies/ACKs
self.sent_ncAckBfr = {}
self.incoming_moduleMsgBfr = Queue.Queue(maxsize=1000)
# Stores the sequence nos which are received but unacknowledged
self.unprocessed_gnSeqNos = {}
###############################
self.error_scope = 255
self.default_seq_no = 0
self.upper_seq_bytes_limit = 16777215
# Specifies the no of bytes after which the subseq_no starts
self.seq_no_partition_size = 3
self.initial_session_id = self.default_seq_no
self.gn_session_id = {}
self.nc_session_id = self.initialize_nc_session_id()
###############################
"""
# Below are dictionaries storing each guest node, cloud's details related
# to sequence nos
# key=inst_id
"""
# subseq_no of the last msg (excluding duplicate) sent to NC by GN
self.highest_gn_subseq_no = {}
# subseq_no of the last msg (excluding duplicate) sent to GN by NC
self.highest_nc_subseq_no = {}
# subseq_no of the last msg of GN (excluding duplicate) acknowledged by NC
self.ackd_gn_subseq_no = {}
# subseq_no of the last msg of NC (excluding duplicate) acknowledged by GN
self.ackd_nc_subseq_no = {}
##############################
# currently all tests are done using same window_size for GN and NC
self.gn_window_size = 1
self.nc_window_size = 1
# to save global msg_processor's address
self.msg_processor = ''
# keeps track of all registered GNs
self.registered_nodes = []
# Dict maintaining gn_id and socket mapping
self.gn_socket_mapping = {}
logger.debug(self.thread_name+" Initialized.")
except Exception as inst:
logger.critical("Exception in init(): " + str(inst))
def process_data_msg(self, item):
logger.debug("DATA MSG Received..................................")
if item.msg != None:
# sends msg to bufr mngr to send it to cloud
self.send_data_msg('cloud', item.msg)
# sends msg to bufr mngr to send ACK to GN
self.send_ack(item.seq_no, item.sock_or_gn_id, 0)
logger.debug("Data ACK sent to buffer_mngr.")
def __init__(self, thread_name, port_for_gn):
# can be used by logging module for printing messages related to this thread
self.thread_name = "Thread_" + thread_name
self.incoming_moduleMsgBfr = Queue.Queue(maxsize=1000)
self.port_for_gn = port_for_gn
self.buffer_mngr = ''
self.nc_server = ''
logger.debug(self.thread_name+" Initialized.")
def collect_incoming_data(self, data):
if self.shutdown == 0:
try:
logger.debug("Data from GN being received."+"\n\n")
self.input_buffer.append(data)
except Exception as inst:
logger.critical("Exception in collect_data: " + str(inst)+"\n\n")
else:
logger.critical("Socket Closed."+"\n\n")
def process_external_msg(self, item):
logger.debug("GN msg being processed.")
if item.msg_type == registration_type:
self.process_gn_registration_msg(item)
elif item.msg_type == data_type:
self.process_data_msg(item)
elif item.msg_type == command_type:
self.process_cmd_msg(item)
else:
logger.critical("Unknown Msg type received......")
def __init__(self, thread_name, socket_type, port_for_gn):
asyncore.dispatcher.__init__(self)
threading.Thread.__init__(self)
# can be used by logging module for printing messages related to this thread
self.thread_name = "Thread_" + thread_name
self.daemon = True
self.port_for_gn = port_for_gn
self.buffer_mngr= ''
self.create_socket(socket_type, socket.SOCK_STREAM)
logger.debug(self.thread_name+" Initialized."+"\n\n")
def __init__(self, gn_socket_conn, gn_addr, buffer_mngr):
asynchat.async_chat.__init__(self, gn_socket_conn)
self.gn_addr = gn_addr
self.input_buffer = []
self.output_buffer = ""
self.buffer_mngr = buffer_mngr
# Used to signal that the GN is down so stop receiving.
self.shutdown = 0
self.set_terminator(asynchat_msg_terminator)
logger.debug("Internal Communicator for new GN initialized"+"\n\n")
def initialize_nc_session_id(self):
try:
session_id = self.get_old_session_id(None, 'NC Session ID')
if not session_id:
session_id = self.initial_session_id
session_id = int (session_id)
session_id = self.increment_no(session_id)
self.save_session_id(None, "NC Session ID", session_id)
logger.debug("Seq_no. initialized.\n\n")
return session_id
except Exception as inst:
logger.critical("Exception in initialize_nc_session_id: " + str(inst))
def found_terminator(self):
if self.shutdown == 0:
try:
# logger.critical("Msg received from GN:"+str('%0.4f' % time.time())\
# +str(self.input_buffer)+"\n\n") #+
logger.critical("Msg received from GN:"+str('%0.4f' % time.time())+"<<<<<<<<<<<<<<<<<<<<")
self.handle_request()
logger.debug("Incoming Msg handled."+"\n\n")
self.input_buffer = []
except Exception as inst:
logger.critical("Exception in found_terminator: " + str(inst))
else:
logger.critical("Socket Closed."+"\n\n")
def process_gn_registration_msg(self, item):
logger.debug("REGISTRATION Msg Received...........................")
if item.msg != None:
with config_file_lock:
logger.critical("Config file lock acquired------------------------------------------------------------------\n\n")
config = ConfigObj(config_file_name)
# if GN is not registered
if item.sock_or_gn_id not in config["GN Info"]:
self.register_gn(item.msg)
self.send_reg_msg_to_cloud('cloud')
logger.critical("Config file lock released------------------------------------------------------------------\n\n")
self.send_ack(item.seq_no, item.sock_or_gn_id, 0, str(int(time.time())))
logger.debug("REGISTRATION ACK sent to buffer_mngr.")
def get_socket_obj(self, gn_id):
try:
logger.debug("Socket object corresponding to specific GN retrieved."+ "\n")
if gn_id in self.gn_socket_mapping:
# Lock acquired to access global list variable and released when the with block ends
with gn_socket_list_lock:
# If live socket then return
if self.gn_socket_mapping[gn_id] in gn_socket_list:
return self.gn_socket_mapping[gn_id]
else:
del self.gn_socket_mapping[gn_id]
return None
except Exception as inst:
logger.critical("Exception in get_socket_obj: " + str(inst)+ "\n")
def handle_request(self):
if self.shutdown == 0:
msg = ''
try:
# recreates msg by concatenatning list's elements
for single_msg in self.input_buffer:
msg = msg + single_msg
msg = buffered_msg(None, None, None, msg, self)
# Sends msg to the buffer_mngr's buffer
add_to_thread_buffer(self.buffer_mngr.incoming_moduleMsgBfr,\
msg, "GN_msgs_buffer_mngr")
logger.debug("Msg sent to buffer_mngr."+"\n\n")
except Exception as inst:
logger.critical("Exception in handle_request: " + str(inst))
else:
logger.critical("Socket Closed."+"\n\n")
def send_reg_msg_to_cloud(self, inst_id):
try:
reg_payload = RegistrationPayload()
config = ConfigObj(config_file_name)
l=lambda:defaultdict(l)
reg_dict = l()
if config["Registered"] == 'NO':
reg_dict["Systems Info"] = config["Systems Info"]
for node in config["GN Info"]:
if config["GN Info"][node]["Registered"] == 'NO':
# append this GN's info to the registration dict
reg_dict["GN Info"][node]["Systems Info"] = config["GN Info"][node]["Systems Info"]
reg_dict["GN Info"][node]["Sensors Info"] = config["GN Info"][node]["Sensors Info"]
reg_payload.sys_info = reg_dict
reg_payload.instance_id = get_instance_id()
buff_msg = buffered_msg(registration_type, None, no_reply, [reg_payload], inst_id)
# Sends registration msg by sending to buffer_mngr
add_to_thread_buffer(self.buffer_mngr.outgoing_ncMsgBfr[inst_id], buff_msg, 'buffer_mngr')
logger.debug("Registration msg sent to bufr mngr to send to cloud.")
except Exception as inst:
logger.critical("Exception in send_reg_msg_to_cloud:" + str(inst))
def process_incoming_moduleMsg(self):
try:
if not self.incoming_moduleMsgBfr.empty():
item = self.incoming_moduleMsgBfr.get()
logger.debug("Msg from GN:" + str(item.msg) )
try:
decoded_msg = Message.decode(item.msg)
except Exception as inst:
logger.critical("Exception while decoding msg: " + str(inst) )
logger.critical("So discarding msg............." + str(inst) )
self.incoming_moduleMsgBfr.task_done()
return
inst_id = decoded_msg.header.instance_id
# check for the authenticity of the sender
if (not self.new_node(decoded_msg.header.instance_id)) | \
(decoded_msg.header.message_type == registration_type):
self.map_socket(inst_id, item.sock_or_gn_id)
# check whether the msg is new or duplicate
if self.new_msg(decoded_msg):
# data/registration msg obtained so send it to msg_processor
if decoded_msg.header.message_type == data_type or decoded_msg.header.message_type == registration_type:
session_id = self.convert_to_int(decoded_msg.header.sequence_id[:self.seq_no_partition_size])
subseq_no = self.convert_to_int(decoded_msg.header.sequence_id[self.seq_no_partition_size:])
# prepare the msg in proper format to insert into msg_processor's incoming_moduleMsgBfr
item = buffered_msg(decoded_msg.header.message_type, decoded_msg.header.sequence_id, \
decoded_msg.header.reply_to_id, decoded_msg.payloads, decoded_msg.header.instance_id)
# save the highest_gn_subseq_no temporarily until the msg is processed
# and the ACK (received from the msg_processor thread) is saved in outgoing_moduleAckBfr
self.unprocessed_gnSeqNos[inst_id].append((session_id, subseq_no))
# send to msg_processor's incoming_moduleMsgBfr
add_to_thread_buffer(self.msg_processor.incoming_moduleMsgBfr, item, 'Msg_Processor')
#print "sent to msg processor"+str('%0.4f' % time.time())
else:
logger.critical("OLD MSG DISCARDED.............\n\n")
else:
logger.critical("UNKNOWN GN SO MSG DISCARDED.............")
self.incoming_moduleMsgBfr.task_done()
except Exception as inst:
logger.critical("Exception in process_incoming_moduleMsg: " + str(inst))
def run(self):
try:
logger.debug("Starting " + self.thread_name)
self.store_system_info()
self.buffer_mngr = buffer_mngr_class("buffer_mngr")
self.nc_server = nc_server_class("nc_server_class", socket.AF_INET, self.port_for_gn)
self.buffer_mngr.pass_thread_address(self)
self.nc_server.pass_thread_address(self.buffer_mngr)
self.buffer_mngr.start()
self.nc_server.start()
logger.critical("All threads Started:"+str('%0.4f' % time.time()))
wait_time_set=0;
while True:
if not self.incoming_moduleMsgBfr.empty():
#print "GN DATA MSG^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"+str('%0.4f' % time.time())
item = self.incoming_moduleMsgBfr.get()
logger.debug("Msg from GN received.")
self.process_external_msg(item)
self.incoming_moduleMsgBfr.task_done()
time.sleep(0.0001)
wait_time_set=0
#print "DATA MSG PROCESSED^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"+str('%0.4f' % time.time())
else:
if wait_time_set==0:
wait_time = time.time() + wait_time_for_next_msg
wait_time_set=1;
if wait_time > time.time():
time.sleep(0.0001)
else:
time.sleep(0.1)
except Exception as inst:
logger.critical("Exception in main: " + str(inst))
finally:
self.buffer_mngr.join(1)
self.nc_server.handle_close()
self.nc_server.join(1)
logger.critical("All child threads exited. Parent Exiting...")
def gen_msg(self, item):
try:
header = MessageHeader()
header.message_type = item.msg_type
header.instance_id = get_instance_id()
# saves the session_id + new subseq_no which is called: highest_nc_subseq_no
header.sequence_id = self.gen_nc_seq_no(item.sock_or_gn_id)
# To indicate the last subseq_no of NC till which all ACKs ie replies from this GN have been received
# NOTE: Its important to increment the highest_nc_subseq_no first
# (which is done in above function call: gen_nc_seq_no) and then call gen_nc_ackd_subseq_no
header.user_field1 = self.gen_nc_ackd_subseq_no(item.sock_or_gn_id, item.msg_type)
header.reply_to_id = item.reply_id
msg = Message()
msg.header = header
for each_msg in item.msg:
msg.append(each_msg)
try:
msg = msg.encode()
except Exception as inst:
logger.critical("Exception while encoding msg: " + str(inst) )
logger.debug("Msg Encoded.")
return msg
except Exception as inst:
logger.critical("Exception in gen_msg: " + str(inst))
def new_node(self, inst_id):
try:
if inst_id in self.registered_nodes:
logger.debug("Node is already known.")
return False
# Lock acquired
with config_file_lock:
logger.critical("Config file lock acquired.............\n\n")
# Check in config file for the inst_id
config = ConfigObj(config_file_name)
if inst_id in config['GN Info']:
logger.debug("Node is already known.")
self.registered_nodes.append(inst_id)
logger.critical("Config file lock released.............\n\n")
return False
# Lock released
logger.critical("Config file lock released.............\n\n")
logger.debug("New node.")
return True
except Exception as inst:
logger.critical("Exception in new_node: " + str(inst))
def pass_thread_address(self, buffer_mngr):
self.buffer_mngr = buffer_mngr
logger.debug("buffer address of gn_buffer_mngr saved."+"\n\n")
def pass_thread_address(self, msg_processor):
try:
self.msg_processor = msg_processor
logger.debug("Address of msg processor saved.")
except Exception as inst:
logger.critical("Exception in pass_thread_address: " + str(inst))
def send_data_msg(self, inst_id, data_payloads):
buff_msg = buffered_msg(data_type, None, no_reply, data_payloads, inst_id)
add_to_thread_buffer(self.buffer_mngr.outgoing_ncMsgBfr[inst_id], buff_msg, 'buffer_mngr')
logger.debug("Data msg sent to bufr mngr to send to cloud/GN.")
def new_msg(self, msg):
try:
ret_val = False
inst_id = msg.header.instance_id
old_session_id = self.get_old_session_id(inst_id, "GN Session ID")
new_session_id = self.convert_to_int(msg.header.sequence_id[:self.seq_no_partition_size])
new_subseq_no = self.convert_to_int(msg.header.sequence_id[self.seq_no_partition_size:])
# if any saved session id then check whether saved one and new one match
if old_session_id == new_session_id:
# GN and NC are both up since they last contacted each other
# check if the GN's inst_id in the self.highest_gn_subseq_no dict is present or not
if inst_id in self.highest_gn_subseq_no:
if (self.in_expected_subseq_range(new_subseq_no, inst_id)):
match = 0
# check for duplicate old msg whose ACK is already sent
for response in self.sent_ncAckBfr[inst_id]:
# comparing the seq_no of new msg with reply_id of bfrd response
if response[0] == new_session_id and response[1] == new_subseq_no:
logger.critical("ACK Lost so old msg with\
subseq_no:"+str(new_subseq_no)+ " received.............\n\n")
# resend response
self.send_msg_to_gn(inst_id, response[2])
match = 1
if not match:
# check if the msg has been already received
# but not responded so it does not have any entry in bfrd_responses
for seq_no in self.unprocessed_gnSeqNos[inst_id]:
# comparing the subseq_no of new msg with
# saved subseq_no not yet acknowledged
if seq_no[0] == new_session_id and seq_no[1] == new_subseq_no:
logger.critical("Old msg with subseq_no:\
"+str(new_subseq_no)+ " received whose\
ACK is not yet sent.............\n\n")
match = 1
if not match:
# else its a new msg
ret_val = True
else:
logger.critical("Unexpected subseq_no received: "+\
str(new_subseq_no)+".............\n\n")
# GN is up but NC went down since they last contacted \
# eachother so no record of subseq_no found
else:
logger.critical("Expected session_id received.............\n\n")
self.init_nc_related_node_data_structures(inst_id)
self.init_node_specific_data_structures(inst_id)
# save new GN session_id
self.gn_session_id[inst_id] = new_session_id
ret_val = True
# check whether new session id falls in the expected range and the new seq_no is 1
elif self.valid_new_session_id(old_session_id, new_session_id) and new_subseq_no == 1:
logger.critical("Valid new session_id received.............\n\n")
self.init_nc_related_node_data_structures(inst_id)
self.init_node_specific_data_structures(inst_id)
# save new GN session_id
self.gn_session_id[inst_id] = new_session_id
self.save_session_id(inst_id, "GN Session ID", self.gn_session_id[inst_id])
ret_val = True
else:
logger.critical("Unexpected seq_no received: "+str(new_session_id)+"\t"+\
str(new_subseq_no)+".............\n\n")
if ret_val:
# For this GN: update current ackd_gn_subseq_no and highest_gn_subseq_no based on the new received nos
new_highest_gn_subseq_no = self.convert_to_int(msg.header.sequence_id[self.seq_no_partition_size:])
new_ackd_gn_subseq_no = self.convert_to_int(msg.header.user_field1)
self.highest_gn_subseq_no[inst_id] = self.get_new_no(new_highest_gn_subseq_no, self.highest_gn_subseq_no[inst_id])
new_ackd_gn_subseq_no = self.get_new_no(new_ackd_gn_subseq_no, self.ackd_gn_subseq_no[inst_id])
if self.ackd_gn_subseq_no[inst_id] != new_ackd_gn_subseq_no:
self.ackd_gn_subseq_no[inst_id] = new_ackd_gn_subseq_no
# discard all responses whose ACKs are received
self.discard_ackd_responses(inst_id)
else:
logger.critical("Unexpected seq_no received: "+str(new_subseq_no)+\
": when highest_gn_subseq_no: "+str(self.highest_gn_subseq_no[inst_id])\
+": ackd_gn_subseq_no: "+str(self.ackd_gn_subseq_no[inst_id]))
logger.critical("new_session_id: "+str(new_session_id)+\
": old_session_id: "+str(old_session_id))
logger.debug("new_subseq_no:"+str(new_subseq_no))
logger.debug("highest_gn_subseq_no[inst_id]"+str(self.highest_gn_subseq_no[inst_id]))
logger.debug("ackd_gn_subseq_no[inst_id]"+str(self.ackd_gn_subseq_no[inst_id]))
return ret_val
except Exception as inst:
logger.critical("Exception in new_msg: " + str(inst))
