Пример #1
0
 def send_ncAcks(self, filled_outgoingBfr_ids):
     try:
         for inst_id in filled_outgoingBfr_ids:
             logger.debug("Length of in_to_out bfr:"+str(self.outgoing_ncAckBfr[inst_id].qsize()))
             item = self.outgoing_ncAckBfr[inst_id].get()
             encoded_msg = self.gen_msg(item)
             encoded_msg = encoded_msg + asynchat_msg_terminator
             # reply_id indicates the seq_no of the msg whose ACK is being sent
             # extract the session_id and subseq_no from reply_id
             session_id = self.convert_to_int(item.reply_id[:self.seq_no_partition_size])
             subseq_no = self.convert_to_int(item.reply_id[self.seq_no_partition_size:])
             for msg_id in self.unprocessed_gnSeqNos[inst_id]:
                 if msg_id[0] == session_id and msg_id[1] == subseq_no:
                     self.unprocessed_gnSeqNos[inst_id].remove(msg_id)
             # Check whether this is not a response for the old msg with old session_id
             #if session_id == self.gn_session_id[inst_id]:
             self.sent_ncAckBfr[inst_id].append((\
             session_id, subseq_no, encoded_msg))
             # as its an old msg don't send it
             #else:
                 #logger.critical("Session id of GN: "+str(inst_id)+\
                 #" has changed so discarding msg.............\n\n")
                 #send_msg = 0
             #if send_msg:
             self.send_msg_to_gn(inst_id, encoded_msg)
             self.outgoing_ncAckBfr[inst_id].task_done()
     
     except Exception as inst:
         logger.critical("Exception in send_ncAcks: " + str(inst))
Пример #2
0
 def run(self):
     try:
         # initializes data structures for cloud
         self.init_nc_related_node_data_structures('cloud')
         logger.debug("Starting " + self.thread_name)
         wait_time_set=0
         while True:
             filled_outgoingBfr_ids=self.get_filled_outgoing_bfr_ids()
             while (not self.incoming_moduleMsgBfr.empty()) or (not self.outgoing_ncMsgBfr['cloud'].empty()) or filled_outgoingBfr_ids:
                 self.process_incoming_moduleMsg()
                 if not self.outgoing_ncMsgBfr['cloud'].empty():
                     item = self.outgoing_ncMsgBfr['cloud'].get()
                     encoded_msg = self.gen_msg(item)
                     self.send_msg_to_cloud(encoded_msg)
                 self.send_ncAcks(filled_outgoingBfr_ids)
                 filled_outgoingBfr_ids=[]
                 time.sleep(0.0001)
                 wait_time_set=0
             else:
                 if wait_time_set==0: 
                     wait_time = time.time() + wait_time_for_next_msg
                     wait_time_set=1;
                 if wait_time > time.time():
                     time.sleep(0.0001)
                 else:
                     time.sleep(0.1)
     except Exception as inst:
         logger.critical("Exception in gn_msgs_bufr_mngr run: " + str(inst))
Пример #3
0
 def __init__(self, thread_name):
     try:
         threading.Thread.__init__(self)
         # can be used by logging module for printing messages related to this thread
         self.thread_name = "Thread_" + thread_name
         self.daemon = True
         self.log_file_name = "session_ids"
         #self.handler_vector_table = {}
         #self.initialize_handler_vector_table()
         ###############################
         """
         # Below are dictionaries representing different types of buffers
         # for each GN
         # key=inst_id
         """
         self.outgoing_ncMsgBfr = {}
         self.sent_ncMsgBfr = {}           # for future use
         self.outgoing_ncAckBfr = {}
         # Stores only sent replies/ACKs
         self.sent_ncAckBfr = {}
         self.incoming_moduleMsgBfr = Queue.Queue(maxsize=1000)
         # Stores the sequence nos which are received but unacknowledged
         self.unprocessed_gnSeqNos = {}
         ###############################
         self.error_scope = 255
         self.default_seq_no = 0
         self.upper_seq_bytes_limit = 16777215
         # Specifies the no of bytes after which the subseq_no starts
         self.seq_no_partition_size = 3
         self.initial_session_id = self.default_seq_no
         self.gn_session_id = {}
         self.nc_session_id = self.initialize_nc_session_id()
         ###############################
         """
         # Below are dictionaries storing each guest node, cloud's details related
         # to sequence nos
         # key=inst_id
         """
         # subseq_no of the last msg (excluding duplicate) sent to NC by GN
         self.highest_gn_subseq_no = {}
         # subseq_no of the last msg (excluding duplicate) sent to GN by NC
         self.highest_nc_subseq_no = {}
         # subseq_no of the last msg of GN (excluding duplicate) acknowledged by NC
         self.ackd_gn_subseq_no = {}
         # subseq_no of the last msg of NC (excluding duplicate) acknowledged by GN
         self.ackd_nc_subseq_no = {}
         ##############################
         # currently all tests are done using same window_size for GN and NC
         self.gn_window_size = 1
         self.nc_window_size = 1
         # to save global msg_processor's address
         self.msg_processor = ''
         # keeps track of all registered GNs
         self.registered_nodes = []
         # Dict maintaining gn_id and socket mapping
         self.gn_socket_mapping = {}
         logger.debug(self.thread_name+" Initialized.")
     except Exception as inst:
         logger.critical("Exception in init(): " + str(inst))
Пример #4
0
 def process_data_msg(self, item):
     logger.debug("DATA MSG Received..................................")
     if item.msg != None:
         # sends msg to bufr mngr to send it to cloud
         self.send_data_msg('cloud', item.msg)
     # sends msg to bufr mngr to send ACK to GN
     self.send_ack(item.seq_no, item.sock_or_gn_id, 0)
     logger.debug("Data ACK sent to buffer_mngr.")
Пример #5
0
 def __init__(self, thread_name, port_for_gn):
     # can be used by logging module for printing messages related to this thread
     self.thread_name = "Thread_" + thread_name
     self.incoming_moduleMsgBfr = Queue.Queue(maxsize=1000)
     self.port_for_gn = port_for_gn
     self.buffer_mngr =  ''
     self.nc_server = ''
     logger.debug(self.thread_name+" Initialized.")
 def collect_incoming_data(self, data):
     if self.shutdown == 0:
         try:
             logger.debug("Data from GN being received."+"\n\n")
             self.input_buffer.append(data)                                                                          
         except Exception as inst:
             logger.critical("Exception in collect_data: " + str(inst)+"\n\n")
     else:
         logger.critical("Socket Closed."+"\n\n")
Пример #7
0
 def process_external_msg(self, item):
     logger.debug("GN msg being processed.")
     if item.msg_type == registration_type:
         self.process_gn_registration_msg(item)
     elif item.msg_type == data_type:
         self.process_data_msg(item)
     elif item.msg_type == command_type:
         self.process_cmd_msg(item)
     else:
         logger.critical("Unknown Msg type received......")
Пример #8
0
 def __init__(self, thread_name, socket_type, port_for_gn):
     asyncore.dispatcher.__init__(self)
     threading.Thread.__init__(self)
     # can be used by logging module for printing messages related to this thread
     self.thread_name = "Thread_" + thread_name
     self.daemon = True
     self.port_for_gn = port_for_gn
     self.buffer_mngr= ''
     self.create_socket(socket_type, socket.SOCK_STREAM)
     logger.debug(self.thread_name+" Initialized."+"\n\n")
 def __init__(self, gn_socket_conn, gn_addr, buffer_mngr):
     asynchat.async_chat.__init__(self, gn_socket_conn)
     self.gn_addr = gn_addr                                                                                  
     self.input_buffer = []                                                                                  
     self.output_buffer = ""                                                                                 
     self.buffer_mngr = buffer_mngr
     # Used to signal that the GN is down so stop receiving.  
     self.shutdown = 0
     self.set_terminator(asynchat_msg_terminator)                                                                         
     logger.debug("Internal Communicator for new GN initialized"+"\n\n")
Пример #10
0
 def initialize_nc_session_id(self):
     try:
         session_id = self.get_old_session_id(None, 'NC Session ID')
         if not session_id:
             session_id = self.initial_session_id
         session_id = int (session_id)
         session_id = self.increment_no(session_id)
         self.save_session_id(None, "NC Session ID", session_id)
         logger.debug("Seq_no. initialized.\n\n")
         return session_id
     except Exception as inst:
         logger.critical("Exception in initialize_nc_session_id: " + str(inst))
 def found_terminator(self):
     if self.shutdown == 0:
         try:
             # logger.critical("Msg received from GN:"+str('%0.4f' % time.time())\
             # +str(self.input_buffer)+"\n\n") #+
             logger.critical("Msg received from GN:"+str('%0.4f' % time.time())+"<<<<<<<<<<<<<<<<<<<<")
             self.handle_request()
             logger.debug("Incoming Msg handled."+"\n\n")
             self.input_buffer = []
         except Exception as inst:
             logger.critical("Exception in found_terminator: " + str(inst))
     else:
         logger.critical("Socket Closed."+"\n\n")
Пример #12
0
 def process_gn_registration_msg(self, item):
     logger.debug("REGISTRATION Msg Received...........................")
     if item.msg != None:
         with config_file_lock:
             logger.critical("Config file lock acquired------------------------------------------------------------------\n\n")
             config = ConfigObj(config_file_name)
             # if GN is not registered
             if item.sock_or_gn_id not in config["GN Info"]:
                 self.register_gn(item.msg)
                 self.send_reg_msg_to_cloud('cloud')
             logger.critical("Config file lock released------------------------------------------------------------------\n\n")
     self.send_ack(item.seq_no, item.sock_or_gn_id, 0, str(int(time.time())))
     logger.debug("REGISTRATION ACK sent to buffer_mngr.")
Пример #13
0
 def get_socket_obj(self, gn_id):
     try:
         logger.debug("Socket object corresponding to specific GN retrieved."+ "\n")
         if gn_id in self.gn_socket_mapping:
             # Lock acquired to access global list variable and released when the with block ends
             with gn_socket_list_lock:
                 # If live socket then return
                 if self.gn_socket_mapping[gn_id] in gn_socket_list:
                     return self.gn_socket_mapping[gn_id]
                 else:
                     del self.gn_socket_mapping[gn_id]
         return None
     except Exception as inst:
         logger.critical("Exception in get_socket_obj: " + str(inst)+ "\n")
 def handle_request(self):
     if self.shutdown == 0:
         msg = ''
         try:
             # recreates msg by concatenatning list's elements
             for single_msg in self.input_buffer:
                 msg = msg + single_msg                                              
             msg = buffered_msg(None, None, None, msg, self)
             # Sends msg to the buffer_mngr's buffer
             add_to_thread_buffer(self.buffer_mngr.incoming_moduleMsgBfr,\
             msg, "GN_msgs_buffer_mngr")                                             
             logger.debug("Msg sent to buffer_mngr."+"\n\n")
         except Exception as inst:
             logger.critical("Exception in handle_request: " + str(inst))
     else:
         logger.critical("Socket Closed."+"\n\n")
Пример #15
0
 def send_reg_msg_to_cloud(self, inst_id):
     try:
         reg_payload = RegistrationPayload()
         config = ConfigObj(config_file_name)
         l=lambda:defaultdict(l)
         reg_dict = l()
         if config["Registered"] == 'NO':
             reg_dict["Systems Info"] = config["Systems Info"]
         for node in config["GN Info"]:
             if config["GN Info"][node]["Registered"] == 'NO':
                 # append this GN's info to the registration dict
                 reg_dict["GN Info"][node]["Systems Info"] = config["GN Info"][node]["Systems Info"]
                 reg_dict["GN Info"][node]["Sensors Info"] = config["GN Info"][node]["Sensors Info"]
         reg_payload.sys_info = reg_dict
         reg_payload.instance_id = get_instance_id()
         buff_msg = buffered_msg(registration_type, None, no_reply, [reg_payload], inst_id)
         # Sends registration msg by sending to buffer_mngr
         add_to_thread_buffer(self.buffer_mngr.outgoing_ncMsgBfr[inst_id], buff_msg, 'buffer_mngr')
         logger.debug("Registration msg sent to bufr mngr to send to cloud.")
     except Exception as inst:
         logger.critical("Exception in send_reg_msg_to_cloud:" + str(inst))
Пример #16
0
 def process_incoming_moduleMsg(self):
     try:
         if not self.incoming_moduleMsgBfr.empty():
             item = self.incoming_moduleMsgBfr.get()
             logger.debug("Msg from GN:"  +  str(item.msg) )
             try:
                 decoded_msg = Message.decode(item.msg)
             except Exception as inst:
                 logger.critical("Exception while decoding msg: " + str(inst) )
                 logger.critical("So discarding msg............." + str(inst) )
                 self.incoming_moduleMsgBfr.task_done()
                 return 
             inst_id = decoded_msg.header.instance_id
             # check for the authenticity of the sender
             if (not self.new_node(decoded_msg.header.instance_id)) | \
             (decoded_msg.header.message_type == registration_type):
                 self.map_socket(inst_id, item.sock_or_gn_id)
                 # check whether the msg is new or duplicate
                 if self.new_msg(decoded_msg):
                     # data/registration msg obtained so send it to msg_processor
                     if decoded_msg.header.message_type == data_type or decoded_msg.header.message_type == registration_type:
                         session_id = self.convert_to_int(decoded_msg.header.sequence_id[:self.seq_no_partition_size])
                         subseq_no = self.convert_to_int(decoded_msg.header.sequence_id[self.seq_no_partition_size:])
                         # prepare the msg in proper format to insert into msg_processor's incoming_moduleMsgBfr
                         item = buffered_msg(decoded_msg.header.message_type, decoded_msg.header.sequence_id, \
                         decoded_msg.header.reply_to_id, decoded_msg.payloads, decoded_msg.header.instance_id)
                         # save the highest_gn_subseq_no temporarily until the msg is processed 
                         # and the ACK (received from the msg_processor thread) is saved in outgoing_moduleAckBfr 
                         self.unprocessed_gnSeqNos[inst_id].append((session_id, subseq_no))
                         # send to msg_processor's incoming_moduleMsgBfr
                         add_to_thread_buffer(self.msg_processor.incoming_moduleMsgBfr, item, 'Msg_Processor')
                         #print "sent to msg processor"+str('%0.4f' % time.time())
                 else:
                     logger.critical("OLD MSG DISCARDED.............\n\n")
             else:
                 logger.critical("UNKNOWN GN SO MSG DISCARDED.............")
             self.incoming_moduleMsgBfr.task_done()
         
     except Exception as inst:
         logger.critical("Exception in process_incoming_moduleMsg: " + str(inst))
Пример #17
0
    def run(self):
        try:
            logger.debug("Starting " + self.thread_name)
            self.store_system_info()
            self.buffer_mngr = buffer_mngr_class("buffer_mngr")
            self.nc_server = nc_server_class("nc_server_class", socket.AF_INET, self.port_for_gn)
            self.buffer_mngr.pass_thread_address(self)
            self.nc_server.pass_thread_address(self.buffer_mngr)
            self.buffer_mngr.start()
            self.nc_server.start()
            logger.critical("All threads Started:"+str('%0.4f' % time.time()))
            wait_time_set=0;
            while True:
                if not self.incoming_moduleMsgBfr.empty():
                    #print "GN DATA MSG^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"+str('%0.4f' % time.time())
                    item = self.incoming_moduleMsgBfr.get()
                    logger.debug("Msg from GN received.")
                    self.process_external_msg(item)
                    self.incoming_moduleMsgBfr.task_done()
                    time.sleep(0.0001)
                    wait_time_set=0
                    #print "DATA MSG PROCESSED^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"+str('%0.4f' % time.time())
                else:
                    if wait_time_set==0: 
                        wait_time = time.time() + wait_time_for_next_msg
                        wait_time_set=1;
                    if wait_time > time.time():
                        time.sleep(0.0001)
                    else:
                        time.sleep(0.1)
        except Exception as inst:
            logger.critical("Exception in main: " + str(inst))

        finally:
            self.buffer_mngr.join(1)
            self.nc_server.handle_close()
            self.nc_server.join(1)
            logger.critical("All child threads exited. Parent Exiting...")
Пример #18
0
 def gen_msg(self, item):
     try:
         header = MessageHeader()
         header.message_type = item.msg_type
         header.instance_id = get_instance_id()
         # saves the session_id + new subseq_no which is called: highest_nc_subseq_no
         header.sequence_id = self.gen_nc_seq_no(item.sock_or_gn_id)
         # To indicate the last subseq_no of NC till which all ACKs ie replies from this GN have been received
         # NOTE: Its important to increment the highest_nc_subseq_no first
         # (which is done in above function call: gen_nc_seq_no) and then call gen_nc_ackd_subseq_no
         header.user_field1 = self.gen_nc_ackd_subseq_no(item.sock_or_gn_id, item.msg_type)
         header.reply_to_id = item.reply_id
         msg = Message()
         msg.header = header
         for each_msg in item.msg:
             msg.append(each_msg)
         try:
             msg = msg.encode()
         except Exception as inst:
             logger.critical("Exception while encoding msg: " + str(inst) )
         logger.debug("Msg Encoded.")
         return msg
     except Exception as inst:
         logger.critical("Exception in gen_msg: " + str(inst))
Пример #19
0
 def new_node(self, inst_id):
     try:
         if inst_id in self.registered_nodes:
             logger.debug("Node is already known.")
             return False
         # Lock acquired
         with config_file_lock:
             logger.critical("Config file lock acquired.............\n\n")
             # Check in config file for the inst_id
             config = ConfigObj(config_file_name)
             if inst_id in config['GN Info']:
                 logger.debug("Node is already known.")
                 self.registered_nodes.append(inst_id)
                 logger.critical("Config file lock released.............\n\n")
                 return False
             # Lock released
             logger.critical("Config file lock released.............\n\n")
         logger.debug("New node.")
         return True
     except Exception as inst:
         logger.critical("Exception in new_node: " + str(inst))
Пример #20
0
 def pass_thread_address(self, buffer_mngr):
     self.buffer_mngr = buffer_mngr
     logger.debug("buffer address of gn_buffer_mngr saved."+"\n\n")
Пример #21
0
 def pass_thread_address(self, msg_processor):
     try:
         self.msg_processor = msg_processor
         logger.debug("Address of msg processor saved.")
     except Exception as inst:
         logger.critical("Exception in pass_thread_address: " + str(inst))
Пример #22
0
 def send_data_msg(self, inst_id, data_payloads):
     buff_msg = buffered_msg(data_type, None, no_reply, data_payloads, inst_id)
     add_to_thread_buffer(self.buffer_mngr.outgoing_ncMsgBfr[inst_id], buff_msg, 'buffer_mngr')
     logger.debug("Data msg sent to bufr mngr to send to cloud/GN.")
Пример #23
0
 def new_msg(self, msg):
     try:
         ret_val = False
         inst_id = msg.header.instance_id
         old_session_id = self.get_old_session_id(inst_id, "GN Session ID")
         new_session_id = self.convert_to_int(msg.header.sequence_id[:self.seq_no_partition_size])
         new_subseq_no = self.convert_to_int(msg.header.sequence_id[self.seq_no_partition_size:])
         # if any saved session id then check whether saved one and new one match
         if old_session_id == new_session_id:
             # GN and NC are both up since they last contacted each other
             # check if the GN's inst_id in the self.highest_gn_subseq_no dict is present or not
             if inst_id in self.highest_gn_subseq_no:
                 if (self.in_expected_subseq_range(new_subseq_no, inst_id)):
                     match = 0
                     # check for duplicate old msg whose ACK is already sent
                     for response in self.sent_ncAckBfr[inst_id]:
                         # comparing the seq_no of new msg with reply_id of bfrd response
                         if response[0] == new_session_id and response[1] == new_subseq_no:
                             logger.critical("ACK Lost so old msg with\
                             subseq_no:"+str(new_subseq_no)+ " received.............\n\n")
                             # resend response
                             self.send_msg_to_gn(inst_id, response[2])
                             match = 1
                     if not match:
                         # check if the msg has been already received
                         # but not responded so it does not have any entry in bfrd_responses
                         for seq_no in self.unprocessed_gnSeqNos[inst_id]:
                             # comparing the subseq_no of new msg with
                             # saved subseq_no not yet acknowledged
                             if seq_no[0] == new_session_id and seq_no[1] == new_subseq_no:
                                 logger.critical("Old msg with subseq_no:\
                                 "+str(new_subseq_no)+ " received whose\
                                 ACK is not yet sent.............\n\n")
                                 match = 1
                     if not match:
                         # else its a new msg
                         ret_val = True
                 else:
                     logger.critical("Unexpected subseq_no received: "+\
                     str(new_subseq_no)+".............\n\n")
             # GN is up but NC went down since they last contacted \
             # eachother so no record of subseq_no found
             else:
                 logger.critical("Expected session_id received.............\n\n")
                 self.init_nc_related_node_data_structures(inst_id)
                 self.init_node_specific_data_structures(inst_id)
                 # save new GN session_id
                 self.gn_session_id[inst_id] = new_session_id
                 ret_val = True
         # check whether new session id falls in the expected range and the new seq_no is 1
         elif self.valid_new_session_id(old_session_id, new_session_id) and new_subseq_no == 1:
             logger.critical("Valid new session_id received.............\n\n")
             self.init_nc_related_node_data_structures(inst_id)
             self.init_node_specific_data_structures(inst_id)
             # save new GN session_id
             self.gn_session_id[inst_id] = new_session_id
             self.save_session_id(inst_id, "GN Session ID", self.gn_session_id[inst_id])
             ret_val = True
         else:
             logger.critical("Unexpected seq_no received: "+str(new_session_id)+"\t"+\
                 str(new_subseq_no)+".............\n\n")
         if ret_val:
             # For this GN: update current ackd_gn_subseq_no and highest_gn_subseq_no based on the new received nos 
             new_highest_gn_subseq_no = self.convert_to_int(msg.header.sequence_id[self.seq_no_partition_size:])
             new_ackd_gn_subseq_no = self.convert_to_int(msg.header.user_field1)
             self.highest_gn_subseq_no[inst_id] = self.get_new_no(new_highest_gn_subseq_no, self.highest_gn_subseq_no[inst_id])
             new_ackd_gn_subseq_no = self.get_new_no(new_ackd_gn_subseq_no, self.ackd_gn_subseq_no[inst_id])
             if self.ackd_gn_subseq_no[inst_id] != new_ackd_gn_subseq_no:
                 self.ackd_gn_subseq_no[inst_id] = new_ackd_gn_subseq_no
                 # discard all responses whose ACKs are received
                 self.discard_ackd_responses(inst_id)
         else:
             logger.critical("Unexpected seq_no received: "+str(new_subseq_no)+\
                 ": when highest_gn_subseq_no: "+str(self.highest_gn_subseq_no[inst_id])\
                     +": ackd_gn_subseq_no: "+str(self.ackd_gn_subseq_no[inst_id]))
             logger.critical("new_session_id: "+str(new_session_id)+\
                 ": old_session_id: "+str(old_session_id))
         logger.debug("new_subseq_no:"+str(new_subseq_no))
         logger.debug("highest_gn_subseq_no[inst_id]"+str(self.highest_gn_subseq_no[inst_id]))
         logger.debug("ackd_gn_subseq_no[inst_id]"+str(self.ackd_gn_subseq_no[inst_id]))
         return ret_val
     except Exception as inst:
         logger.critical("Exception in new_msg: " + str(inst))