def main(self): control_thread = [] self.client_threads = {} self.reload_clients = {} self.server_thread = [] self.system_lock = threading.Lock() # find matching client streams for this host client_streams = [] for istream in range(int(self.cfg["NUM_STREAM"])): (req_host, beam_id, subband_id) = Config.getStreamConfig(istream, self.cfg) if req_host == self.req_host: client_streams.append(istream) # find matching server stream server_streams = [] if self.cfg["SERVER_HOST"] == self.req_host: server_streams.append(-1) daemon_states = {} for stream in server_streams: self.log(2, "main: client_thread[-1] = clientThread(-1)") daemon_states[-1] = {} server_thread = clientThread(-1, self, daemon_states[-1]) self.log(2, "main: client_thread[-1].start()") server_thread.start() self.log(2, "main: client_thread[-1] started") sleep(1) # start a control thread for each stream for stream in client_streams: daemon_states[stream] = {} self.log( 2, "main: client_thread[" + str(stream) + "] = clientThread (" + str(stream) + ")") self.reload_clients[stream] = False self.client_threads[stream] = clientThread(stream, self, daemon_states[stream]) self.log(2, "main: client_thread[" + str(stream) + "].start()") self.client_threads[stream].start() self.log(2, "main: client_thread[" + str(stream) + "] started!") # main thread disks_to_monitor = [self.cfg["CLIENT_DIR"]] # create socket for LMC commands sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((self.req_host, int(self.cfg["LMC_PORT"]))) sock.listen(5) can_read = [sock] can_write = [] can_error = [] timeout = 1 hw_poll = 5 counter = 0 sensors = {} # monitor / control loop while not self.quit_event.isSet(): self.log(3, "Main Loop: counter=" + str(counter)) while (counter == 0): self.log(2, "Refreshing monitoring points") self.log(3, "main: getDiskCapacity ()") rval, disks = lmc_mon.getDiskCapacity(disks_to_monitor, DL) self.log(3, "main: " + str(disks)) self.log(3, "main: getLoads()") rval, loads = lmc_mon.getLoads(DL) self.log(3, "main: " + str(loads)) self.log(3, "main: getSMRBCapacity(" + str(client_streams) + ")") rval, smrbs = lmc_mon.getSMRBCapacity(client_streams, self.quit_event, DL) self.log(3, "main: " + str(smrbs)) self.log(3, "main: getIPMISensors()") rval, sensors = lmc_mon.getIPMISensors(DL) self.log(3, "main: " + str(sensors)) counter = hw_poll self.log( 3, "main: calling select len(can_read)=" + str(len(can_read))) timeout = 1 did_read = [] did_write = [] did_error = [] try: did_read, did_write, did_error = select.select( can_read, can_write, can_error, timeout) except select.error as e: self.quit_event.set() else: self.log( 3, "main: read=" + str(len(did_read)) + " write=" + str(len(did_write)) + " error=" + str(len(did_error))) if (len(did_read) > 0): for handle in did_read: if (handle == sock): (new_conn, addr) = sock.accept() self.log(2, "main: accept connection from " + repr(addr)) # add the accepted connection to can_read can_read.append(new_conn) # new_conn.send("Welcome to the LMC interface\r\n") # an accepted connection must have generated some data else: try: raw = handle.recv(4096) except socket.error, e: if e.errno == errno.ECONNRESET: self.log(2, "main: closing connection") handle.close() for i, x in enumerate(can_read): if (x == handle): del can_read[i] else: raise e else: message = raw.strip() self.log(2, "main: message='" + message + "'") if len(message) == 0: self.log(2, "main: closing connection") handle.close() for i, x in enumerate(can_read): if (x == handle): del can_read[i] else: xml = xmltodict.parse(message) command = xml["lmc_cmd"]["command"] if command == "reload_clients": self.log(1, "Reloading clients") for stream in client_streams: self.reload_clients[stream] = True all_reloaded = False while (not self.parent.quit_event.isSet() and not all_reloaded): all_reloaded = True for stream in client_streams: if not self.reload_clients[stream]: all_reloaded = False if not all_reloaded: self.log( 1, "Waiting for clients to reload" ) sleep(1) self.log(1, "Clients reloaded") response = "<lmc_reply>OK</lmc_reply>" if command == "daemon_status": response = "" response += "<lmc_reply>" for stream in server_streams: response += "<stream id='" + str( stream) + "'>" for daemon in daemon_states[ stream].keys(): response += "<daemon name='" + daemon + "'>" + str( daemon_states[stream] [daemon]) + "</daemon>" response += "</stream>" for stream in client_streams: response += "<stream id='" + str( stream) + "'>" for daemon in daemon_states[ stream].keys(): response += "<daemon name='" + daemon + "'>" + str( daemon_states[stream] [daemon]) + "</daemon>" response += "</stream>" response += "</lmc_reply>" elif command == "host_status": response = "<lmc_reply>" for disk in disks.keys(): percent_full = 1.0 - ( float(disks[disk]["available"]) / float(disks[disk]["size"])) response += "<disk mount='" + disk + "' percent_full='" + str( percent_full) + "'>" response += "<size units='MB'>" + disks[ disk]["size"] + "</size>" response += "<used units='MB'>" + disks[ disk]["used"] + "</used>" response += "<available units='MB'>" + disks[ disk]["available"] + "</available>" response += "</disk>" for stream in smrbs.keys(): for key in smrbs[stream].keys(): smrb = smrbs[stream][key] response += "<smrb stream='" + str( stream) + "' key='" + str( key) + "'>" response += "<header_block nbufs='" + str( smrb['hdr'] ['nbufs']) + "'>" + str( smrb['hdr']['full'] ) + "</header_block>" response += "<data_block nbufs='" + str( smrb['data']['nbufs'] ) + "'>" + str(smrb['data']['full'] ) + "</data_block>" response += "</smrb>" response += "<system_load ncore='" + loads[ "ncore"] + "'>" response += "<load1>" + loads[ "1min"] + "</load1>" response += "<load5>" + loads[ "5min"] + "</load5>" response += "<load15>" + loads[ "15min"] + "</load15>" response += "</system_load>" response += "<sensors>" for sensor in sensors.keys(): response += "<metric name='" + sensor + "' units='" + sensors[ sensor]["units"] + "'>" + sensors[ sensor]["value"] + "</metric>" response += "</sensors>" response += "</lmc_reply>" else: response = "<lmc_reply>OK</lmc_reply>" self.log(2, "-> " + response) handle.send(response + "\r\n")
def main (self): control_thread = [] # the threads for each instance of beams, streams and server self.stream_threads = {} self.beam_threads = {} self.server_thread = [] self.reload_beams = {} self.reload_streams = {} self.system_lock = threading.Lock() # find matching client streams for this host self.host_streams = [] for istream in range(int(self.cfg["NUM_STREAM"])): (req_host, beam_id, subband_id) = Config.getStreamConfig (istream, self.cfg) if req_host == self.req_host and not istream in self.host_streams: self.host_streams.append(istream) # find matching client streams for this host self.host_beams = [] for istream in range(int(self.cfg["NUM_STREAM"])): (req_host, beam_id, subband_id) = Config.getStreamConfig (istream, self.cfg) if req_host == self.req_host and not beam_id in self.host_beams: self.host_beams.append(beam_id) # find matching server stream self.host_servers = [] if self.cfg["SERVER_HOST"] == self.req_host: self.host_servers.append(-1) self.server_daemon_states = {} self.stream_daemon_states = {} self.beam_daemon_states = {} # configure disk systems to monitor if len(self.host_servers) > 0: self.disks_to_monitor = [self.cfg["SERVER_DIR"]] else: self.disks_to_monitor = [self.cfg["CLIENT_DIR"]] # gather some initial statistics self.gather_stats() # start server thread for stream in self.host_servers: self.debug("server_thread["+str(stream)+"] = streamThread(-1)") self.server_daemon_states[stream] = {} self.server_thread = serverThread(stream, self, self.server_daemon_states[stream]) self.debug("server_thread["+str(stream)+"].start()") self.server_thread.start() self.debug("server_thread["+str(stream)+"] started") sleep(5) self.log(1, "Server threads started: " + str(len(self.host_servers))) # start a thread for each stream for stream in self.host_streams: self.stream_daemon_states[stream] = {} self.debug("stream_threads["+str(stream)+"] = streamThread ("+str(stream)+")") self.reload_streams[stream] = False self.stream_threads[stream] = streamThread (stream, self, self.stream_daemon_states[stream]) self.debug("stream_threads["+str(stream)+"].start()") self.stream_threads[stream].start() self.debug("stream_thread["+str(stream)+"] started!") self.log(1, "Stream threads started: " + str(len(self.host_streams))) # start a thread for each beam for beam in self.host_beams: self.beam_daemon_states[beam] = {} self.debug("beam_threads["+str(beam)+"] = beamThread ("+str(beam)+")") self.reload_beams[beam] = False self.beam_threads[beam] = beamThread (beam, self, self.beam_daemon_states[beam]) self.debug("beam_threads["+str(beam)+"].start()") self.beam_threads[beam].start() self.debug("beam_thread["+str(beam)+"] started!") self.log(1, "Beam threads started: " + str(len(self.host_beams))) # main thread hw_poll = 5 counter = 0 first_time = True self.debug("starting main loop quit_event=" + str(self.quit_event.isSet())) # control loop while not self.quit_event.isSet(): # update the monitoring points if counter == 0: self.debug("refreshing monitoring points") if not first_time: self.resource_lock.acquire () self.trace("self.gather_stats()") self.gather_stats() first_time = False self.resource_lock.release () self.trace("monitoring points refreshed") counter = hw_poll if not self.quit_event.isSet(): sleep(1) counter -= 1 self.debug("quit_event set, asking lmcThreads to terminate") self.concludeThreads() self.debug("done")