def in_expected_range(self, new_id, old_id): try: # self.error_scope shows the range of session_id which may be old\ # and should be discarded return (new_id > old_id) or self.is_wrap_up(new_id, old_id) except Exception as inst: logger.critical("Exception in in_expected_range: " + str(inst))
def run(self): try: # initializes data structures for cloud self.init_nc_related_node_data_structures('cloud') logger.debug("Starting " + self.thread_name) wait_time_set=0 while True: filled_outgoingBfr_ids=self.get_filled_outgoing_bfr_ids() while (not self.incoming_moduleMsgBfr.empty()) or (not self.outgoing_ncMsgBfr['cloud'].empty()) or filled_outgoingBfr_ids: self.process_incoming_moduleMsg() if not self.outgoing_ncMsgBfr['cloud'].empty(): item = self.outgoing_ncMsgBfr['cloud'].get() encoded_msg = self.gen_msg(item) self.send_msg_to_cloud(encoded_msg) self.send_ncAcks(filled_outgoingBfr_ids) filled_outgoingBfr_ids=[] time.sleep(0.0001) wait_time_set=0 else: if wait_time_set==0: wait_time = time.time() + wait_time_for_next_msg wait_time_set=1; if wait_time > time.time(): time.sleep(0.0001) else: time.sleep(0.1) except Exception as inst: logger.critical("Exception in gn_msgs_bufr_mngr run: " + str(inst))
def get_new_no(self, new_no, old_no): try: if (new_no != old_no) and self.in_expected_range(new_no, old_no): return new_no return old_no except Exception as inst: logger.critical("Exception in get_new_no: " + str(inst))
def send_ncAcks(self, filled_outgoingBfr_ids): try: for inst_id in filled_outgoingBfr_ids: logger.debug("Length of in_to_out bfr:"+str(self.outgoing_ncAckBfr[inst_id].qsize())) item = self.outgoing_ncAckBfr[inst_id].get() encoded_msg = self.gen_msg(item) encoded_msg = encoded_msg + asynchat_msg_terminator # reply_id indicates the seq_no of the msg whose ACK is being sent # extract the session_id and subseq_no from reply_id session_id = self.convert_to_int(item.reply_id[:self.seq_no_partition_size]) subseq_no = self.convert_to_int(item.reply_id[self.seq_no_partition_size:]) for msg_id in self.unprocessed_gnSeqNos[inst_id]: if msg_id[0] == session_id and msg_id[1] == subseq_no: self.unprocessed_gnSeqNos[inst_id].remove(msg_id) # Check whether this is not a response for the old msg with old session_id #if session_id == self.gn_session_id[inst_id]: self.sent_ncAckBfr[inst_id].append((\ session_id, subseq_no, encoded_msg)) # as its an old msg don't send it #else: #logger.critical("Session id of GN: "+str(inst_id)+\ #" has changed so discarding msg.............\n\n") #send_msg = 0 #if send_msg: self.send_msg_to_gn(inst_id, encoded_msg) self.outgoing_ncAckBfr[inst_id].task_done() except Exception as inst: logger.critical("Exception in send_ncAcks: " + str(inst))
def in_expected_subseq_range(self, new_subseq_no, inst_id): try: upper_limit = self.highest_gn_subseq_no[inst_id] + self.gn_window_size # 1. Ex: highest_gn_subseq_no == ackd_gn_subseq_no so acceptable msgs can fall in range # highest_gn_subseq_no < *HERE* <= highest_gn_subseq_no + gn_window_size # 2. Ex: ackd_gn_subseq_no < highest_gn_subseq_no so acceptable msgs can fall in range # ackd_gn_subseq_no < *HERE* <= highest_gn_subseq_no + gn_window_size # 3. Ex: ackd_gn_subseq_no > highest_gn_subseq_no so acceptable msgs can fall in range # ackd_gn_subseq_no < *HERE* <= upper_seq_bytes_limit OR 0 < *HERE* <= (highest_gn_subseq_no + gn_window_size) # Explanation for limit_due_to_wrap_up: suppose upper_seq_bytes_limit=255 and gn_window_size=2 # and highest_gn_subseq_no=254 then upper_limit(calculated above)=256 which is invalid # so set another limit_due_to_wrap_up which will be (in this case): 1 # Wrap-up can occur in both 1 & 2 cases above and 3rd is essentially wrap up if upper_limit > self.upper_seq_bytes_limit: limit_due_to_wrap_up = (self.highest_gn_subseq_no[inst_id] + \ self.gn_window_size) - (self.upper_seq_bytes_limit) upper_limit = self.upper_seq_bytes_limit else: # placeholder value limit_due_to_wrap_up = -1 #return (ackd_gn_subseq_no < new_subseq_no <= upper_limit) or (0 < new_subseq_no <= limit_due_to_wrap_up) if self.highest_gn_subseq_no[inst_id] >= self.ackd_gn_subseq_no[inst_id]: return (self.ackd_gn_subseq_no[inst_id] < new_subseq_no <= upper_limit) or\ (0 < new_subseq_no <= limit_due_to_wrap_up) return (self.ackd_gn_subseq_no[inst_id] < new_subseq_no <= \ self.upper_seq_bytes_limit) or (0 < new_subseq_no <= upper_limit) except Exception as inst: logger.critical("Exception in in_expected_subseq_range: " + str(inst))
def __init__(self, thread_name): try: threading.Thread.__init__(self) # can be used by logging module for printing messages related to this thread self.thread_name = "Thread_" + thread_name self.daemon = True self.log_file_name = "session_ids" #self.handler_vector_table = {} #self.initialize_handler_vector_table() ############################### """ # Below are dictionaries representing different types of buffers # for each GN # key=inst_id """ self.outgoing_ncMsgBfr = {} self.sent_ncMsgBfr = {} # for future use self.outgoing_ncAckBfr = {} # Stores only sent replies/ACKs self.sent_ncAckBfr = {} self.incoming_moduleMsgBfr = Queue.Queue(maxsize=1000) # Stores the sequence nos which are received but unacknowledged self.unprocessed_gnSeqNos = {} ############################### self.error_scope = 255 self.default_seq_no = 0 self.upper_seq_bytes_limit = 16777215 # Specifies the no of bytes after which the subseq_no starts self.seq_no_partition_size = 3 self.initial_session_id = self.default_seq_no self.gn_session_id = {} self.nc_session_id = self.initialize_nc_session_id() ############################### """ # Below are dictionaries storing each guest node, cloud's details related # to sequence nos # key=inst_id """ # subseq_no of the last msg (excluding duplicate) sent to NC by GN self.highest_gn_subseq_no = {} # subseq_no of the last msg (excluding duplicate) sent to GN by NC self.highest_nc_subseq_no = {} # subseq_no of the last msg of GN (excluding duplicate) acknowledged by NC self.ackd_gn_subseq_no = {} # subseq_no of the last msg of NC (excluding duplicate) acknowledged by GN self.ackd_nc_subseq_no = {} ############################## # currently all tests are done using same window_size for GN and NC self.gn_window_size = 1 self.nc_window_size = 1 # to save global msg_processor's address self.msg_processor = '' # keeps track of all registered GNs self.registered_nodes = [] # Dict maintaining gn_id and socket mapping self.gn_socket_mapping = {} logger.debug(self.thread_name+" Initialized.") except Exception as inst: logger.critical("Exception in init(): " + str(inst))
def handle_close(self): try: self.shutdown = 1 self.delete_socket() self.close() logger.critical("Socket Connection with GN closed."+"\n\n") except Exception as inst: logger.critical("Exception in handle_close: " + str(inst)+"\n\n")
def valid_new_session_id(self, old_session_id, new_session_id): try: if old_session_id: return self.in_expected_sessionseq_range(new_session_id, old_session_id) # if this GN is contacting the NC for the first time then accept any session_id from GN return True except Exception as inst: logger.critical("Exception in valid_new_session_id: " + str(inst)) return False
def close_gn_socket(self): # Lock acquired to access global list variable and released when the 'with' block ends with gn_socket_list_lock: for socket in gn_socket_list: socket.shutdown = 1 gn_socket_list.remove(socket) socket.close() logger.info("Individual GN socket closed.") logger.critical("NC's individual GN sockets closed."+"\n\n")
def collect_incoming_data(self, data): if self.shutdown == 0: try: logger.debug("Data from GN being received."+"\n\n") self.input_buffer.append(data) except Exception as inst: logger.critical("Exception in collect_data: " + str(inst)+"\n\n") else: logger.critical("Socket Closed."+"\n\n")
def map_socket(self, inst_id, socket): try: with gn_socket_list_lock: if socket in gn_socket_list: self.gn_socket_mapping[inst_id] = socket elif inst_id in self.gn_socket_mapping and self.gn_socket_mapping[inst_id]==socket: del self.gn_socket_mapping[inst_id] except Exception as inst: logger.critical("Exception in map_socket: " + str(inst)+"\n")
def convert_to_int(self, byte_seq): try: if byte_seq != None: byte_seq = bytearray(byte_seq) int_id = sum(byte_seq[i] << ((len(byte_seq)-1-i) * 8) for i in range(len(byte_seq))) return int_id return None except Exception as inst: logger.critical("Exception in convert_to_int: " + str(inst))
def get_node_info(config_file_name): config_object = ConfigObj(config_file_name) config = config_object["Systems Info"] list_of_keywords = ["hostname", "noOfProcessors", "cpuModelName", "cpuVendorID", "extensions", "hardware", "interfaces", "memTotal", "osName", "osVersion", "osID", "osPrettyName", "osVersionID", "osKernelRelease", "machineName", "kernelVersion", "diskStorage"] list_of_cmds = [ "hostname", "cat /proc/cpuinfo | grep -i processor | wc -l", "cat /proc/cpuinfo | grep -i model\ name | uniq | tr -s ' ' | cut -d ':' -f 2", "cat /proc/cpuinfo | grep -i vendor_id | uniq | cut -d ':' -f 2", ["cat /proc/cpuinfo | grep -i flags | tr ' ' '\n' | sort | uniq | grep -i -v flag | tr '\n' ' '", "cat /proc/cpuinfo | grep -i features | tr ' ' '\n' | sort | uniq | grep -i -v features | tr '\n' ' '"], "cat /proc/cpuinfo | grep -i hardware | cut -d ':' -f 2", ["ls /sys/class/net/", "cat /sys/class/net/"], 'cat /proc/meminfo | tr -s " " | grep -i MemTotal | cut -d ":" -f 2', 'cat /etc/os-release | grep -i NAME | grep -vi "pretty_name" | cut -d "=" -f 2 ', 'cat /etc/os-release | grep -i VERSION | grep -vi "version_id" |cut -d "=" -f 2', 'cat /etc/os-release | grep -i ID=ubuntu | cut -d "=" -f 2', 'cat /etc/os-release | grep -i PRETTY_NAME | cut -d "=" -f 2', 'cat /etc/os-release | grep -i VERSION_ID | cut -d "=" -f 2', 'uname -r', 'uname -m', 'cat /proc/version', "df -h | tr -s ' ' | cut -d ' ' -f 1,2,5,6 | tr '\n' ';' | sed 's/;/\ ;\ /g'" ] for key, cmd in zip(list_of_keywords, list_of_cmds): try: if not isinstance(cmd, list): config[key] = bashit(cmd) else: if key == "interfaces": interfaces = bashit(cmd[0]) interfaces=interfaces.split('\n') config[key] = interfaces if interfaces: interface_details = '' for i in interfaces: if i <> 'lo': interface_details += str(i) + ',' + bashit(cmd[1]+str(i)+"/address") + '\n' interface_details = interface_details.split('\n') config[key] = interface_details elif key == 'extensions': config[key] = bashit(cmd[0]) if not config[key]: config[key] = bashit(cmd[1]) if not config[key]: del config[key] except Exception as inst: logger.critical("Exception in get_system_info: " + str(inst) + "while executing command " + str(cmd) +"\n\n") return 1 config_object.write() return 0
def init_node_specific_data_structures(self, inst_id): try: self.highest_gn_subseq_no[inst_id] = self.default_seq_no self.ackd_gn_subseq_no[inst_id] = self.default_seq_no self.unprocessed_gnSeqNos[inst_id] = [] self.sent_ncAckBfr[inst_id] = [] self.outgoing_ncAckBfr[inst_id] = Queue.Queue(100) except Exception as inst: logger.critical("Exception in init_node_specific_data_structures: "\ + str(inst))
def process_external_msg(self, item): logger.debug("GN msg being processed.") if item.msg_type == registration_type: self.process_gn_registration_msg(item) elif item.msg_type == data_type: self.process_data_msg(item) elif item.msg_type == command_type: self.process_cmd_msg(item) else: logger.critical("Unknown Msg type received......")
def register_gn(self, gn_info): try: for single_gn_info in gn_info: config = ConfigObj(config_file_name) config["GN Info"][single_gn_info.instance_id] = single_gn_info.sys_info config["GN Info"][single_gn_info.instance_id]["Registered"] = 'NO' config.write() logger.info("GN registration info saved in config file.") except Exception as inst: logger.critical("Exception in register_gn:" + str(inst))
def convert_to_bytearray(self, int_no): try: if int_no != None: byte_seq = bytearray([0,0,0]) for i in range(self.seq_no_partition_size): byte_seq[i] = (int_no >> ((self.seq_no_partition_size-1-i)*8)) & 0xff return byte_seq return None except Exception as inst: logger.critical("Exception in convert_to_bytearray: " + str(inst))
def init_nc_related_node_data_structures(self, inst_id): try: # initialize these structures only if they don't exist if inst_id not in self.highest_nc_subseq_no: self.highest_nc_subseq_no[inst_id] = self.default_seq_no self.ackd_nc_subseq_no[inst_id] = self.default_seq_no self.outgoing_ncMsgBfr[inst_id] = Queue.Queue(100) self.sent_ncMsgBfr[inst_id] = [] except Exception as inst: logger.critical("Exception in init_nc_related_node_data_structures:\ " + str(inst))
def gen_nc_seq_no(self, inst_id): try: if inst_id in self.highest_nc_subseq_no: self.highest_nc_subseq_no[inst_id] = self.increment_no(self.highest_nc_subseq_no[inst_id]) logger.info("Sequence no. generated: " + str(self.highest_nc_subseq_no[inst_id]) +\ "for Node:"+str(inst_id)) return str(self.convert_to_bytearray(self.nc_session_id)) +\ str(self.convert_to_bytearray(self.highest_nc_subseq_no[inst_id])) return None except Exception as inst: logger.critical("Exception in gen_nc_seq_no: " + str(inst))
def increment_no(self, int_no): integer_no = copy.copy(int_no) try: if integer_no == self.upper_seq_bytes_limit: # reset it to 1 integer_no = self.default_seq_no + 1 else: integer_no += 1 return integer_no except Exception as inst: logger.critical("Exception in increment_no: " + str(inst))
def delete_socket(self): try: # Lock acquired to access global list variable and released when the 'with' block ends with gn_socket_list_lock: if self in gn_socket_list: gn_socket_list.remove(self) current_socket_list = '' if gn_socket_list: current_socket_list = str(gn_socket_list) logger.critical("Current socket list:"+current_socket_list+"\n") except Exception as inst: logger.critical("Exception in delete_socket: " + str(inst)+"\n")
def initialize_nc_session_id(self): try: session_id = self.get_old_session_id(None, 'NC Session ID') if not session_id: session_id = self.initial_session_id session_id = int (session_id) session_id = self.increment_no(session_id) self.save_session_id(None, "NC Session ID", session_id) logger.debug("Seq_no. initialized.\n\n") return session_id except Exception as inst: logger.critical("Exception in initialize_nc_session_id: " + str(inst))
def process_gn_registration_msg(self, item): logger.debug("REGISTRATION Msg Received...........................") if item.msg != None: with config_file_lock: logger.critical("Config file lock acquired------------------------------------------------------------------\n\n") config = ConfigObj(config_file_name) # if GN is not registered if item.sock_or_gn_id not in config["GN Info"]: self.register_gn(item.msg) self.send_reg_msg_to_cloud('cloud') logger.critical("Config file lock released------------------------------------------------------------------\n\n") self.send_ack(item.seq_no, item.sock_or_gn_id, 0, str(int(time.time()))) logger.debug("REGISTRATION ACK sent to buffer_mngr.")
def handle_accept(self): try: logger.info("New GN connection available." + "\n\n") gn_socket_conn, gn_addr = self.accept() socket = internal_communicator(gn_socket_conn, gn_addr, self.buffer_mngr) # Lock acquired to access global list variable and released when the 'with' block ends with gn_socket_list_lock: # Pass GN's details and new socket's details to a new socket object created for that GN gn_socket_list.append(socket) #logger.debug("Socket object corresponding to new GN created and running.") except: logger.critical("Error in handling the GN connection request."+ "\n\n") pass
def get_socket_obj(self, gn_id): try: logger.debug("Socket object corresponding to specific GN retrieved."+ "\n") if gn_id in self.gn_socket_mapping: # Lock acquired to access global list variable and released when the with block ends with gn_socket_list_lock: # If live socket then return if self.gn_socket_mapping[gn_id] in gn_socket_list: return self.gn_socket_mapping[gn_id] else: del self.gn_socket_mapping[gn_id] return None except Exception as inst: logger.critical("Exception in get_socket_obj: " + str(inst)+ "\n")
def gen_nc_ackd_subseq_no(self, inst_id, msg_type): try: # If bfr_for_sent_msgs corresponding to this GN is empty means # there are no pending unacknowledged *msgs* (not ACKS as ACKS don't require ACK) # sent by NC->this GN AND if this msg is an *ACK* so the ackd_nc_subseq_no # can be safely set equal to highest_nc_subseq_no maintained with this GN # to indicate that NC has received all ACKs for all the msgs sent by it (if any from NC->GN) # or in short NC is not expecting any ACK if msg_type == reply_type and (not self.sent_ncMsgBfr[inst_id]): # assign the recently incremented highest_nc_subseq_no self.ackd_nc_subseq_no[inst_id] = self.highest_nc_subseq_no[inst_id] return str(self.convert_to_bytearray(self.ackd_nc_subseq_no[inst_id])) except Exception as inst: logger.critical("Exception in gen_nc_seq_no: " + str(inst))
def save_session_id(self, inst_id, tag_name, session_id): try: log = ConfigObj(self.log_file_name) if inst_id: # create an entry for the GN if inst_id not in log: log[inst_id] = {} # ex: log[guest_node_id]["GN Session ID"] log[inst_id][tag_name] = session_id else: log[tag_name] = session_id log.write() except Exception as inst: logger.critical("Exception in save_session_id: " + str(inst))
def discard_ackd_responses(self, inst_id): try: for response in self.sent_ncAckBfr[inst_id]: # If session_id doesn't match means its probably an old response so delete it if response[0] != self.gn_session_id[inst_id]: self.sent_ncAckBfr[inst_id].remove(response) # check whether the buffered_response's reply_id ie. GN's subseq_no\ # is now outside GN's current window elif self.response_rcvd_by_gn(self.ackd_gn_subseq_no[inst_id], response[1]): # discard the response self.sent_ncAckBfr[inst_id].remove(response) else: # as ACKS are stored in FIFO order break except Exception as inst: logger.critical("Exception in discard_ackd_responses: " + str(inst)+"\n")
def handle_request(self): if self.shutdown == 0: msg = '' try: # recreates msg by concatenatning list's elements for single_msg in self.input_buffer: msg = msg + single_msg msg = buffered_msg(None, None, None, msg, self) # Sends msg to the buffer_mngr's buffer add_to_thread_buffer(self.buffer_mngr.incoming_moduleMsgBfr,\ msg, "GN_msgs_buffer_mngr") logger.debug("Msg sent to buffer_mngr."+"\n\n") except Exception as inst: logger.critical("Exception in handle_request: " + str(inst)) else: logger.critical("Socket Closed."+"\n\n")
def get_old_session_id(self, inst_id, tag_name): try: log = ConfigObj(self.log_file_name) # check if session_id of a GN is queried if inst_id: # Since NC is up has this GN ever contacted NC? if tag_name == 'GN Session ID' and inst_id in self.gn_session_id: return self.gn_session_id[inst_id] # Has this GN ever contacted NC? if inst_id in log: return int(log[inst_id][tag_name]) # Session ID of NC is queried else: if tag_name in log: return int(log[tag_name]) return None except Exception as inst: logger.critical("Exception in get_old_session_id: " + str(inst)+\ " for:" + str(inst_id) )