def main(self):
control_thread = []
self.client_threads = {}
self.reload_clients = {}
self.server_thread = []
self.system_lock = threading.Lock()
# find matching client streams for this host
client_streams = []
for istream in range(int(self.cfg["NUM_STREAM"])):
(req_host, beam_id,
subband_id) = Config.getStreamConfig(istream, self.cfg)
if req_host == self.req_host:
client_streams.append(istream)
# find matching server stream
server_streams = []
if self.cfg["SERVER_HOST"] == self.req_host:
server_streams.append(-1)
daemon_states = {}
for stream in server_streams:
self.log(2, "main: client_thread[-1] = clientThread(-1)")
daemon_states[-1] = {}
server_thread = clientThread(-1, self, daemon_states[-1])
self.log(2, "main: client_thread[-1].start()")
server_thread.start()
self.log(2, "main: client_thread[-1] started")
sleep(1)
# start a control thread for each stream
for stream in client_streams:
daemon_states[stream] = {}
self.log(
2, "main: client_thread[" + str(stream) +
"] = clientThread (" + str(stream) + ")")
self.reload_clients[stream] = False
self.client_threads[stream] = clientThread(stream, self,
daemon_states[stream])
self.log(2, "main: client_thread[" + str(stream) + "].start()")
self.client_threads[stream].start()
self.log(2, "main: client_thread[" + str(stream) + "] started!")
# main thread
disks_to_monitor = [self.cfg["CLIENT_DIR"]]
# create socket for LMC commands
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((self.req_host, int(self.cfg["LMC_PORT"])))
sock.listen(5)
can_read = [sock]
can_write = []
can_error = []
timeout = 1
hw_poll = 5
counter = 0
sensors = {}
# monitor / control loop
while not self.quit_event.isSet():
self.log(3, "Main Loop: counter=" + str(counter))
while (counter == 0):
self.log(2, "Refreshing monitoring points")
self.log(3, "main: getDiskCapacity ()")
rval, disks = lmc_mon.getDiskCapacity(disks_to_monitor, DL)
self.log(3, "main: " + str(disks))
self.log(3, "main: getLoads()")
rval, loads = lmc_mon.getLoads(DL)
self.log(3, "main: " + str(loads))
self.log(3,
"main: getSMRBCapacity(" + str(client_streams) + ")")
rval, smrbs = lmc_mon.getSMRBCapacity(client_streams,
self.quit_event, DL)
self.log(3, "main: " + str(smrbs))
self.log(3, "main: getIPMISensors()")
rval, sensors = lmc_mon.getIPMISensors(DL)
self.log(3, "main: " + str(sensors))
counter = hw_poll
self.log(
3, "main: calling select len(can_read)=" + str(len(can_read)))
timeout = 1
did_read = []
did_write = []
did_error = []
try:
did_read, did_write, did_error = select.select(
can_read, can_write, can_error, timeout)
except select.error as e:
self.quit_event.set()
else:
self.log(
3, "main: read=" + str(len(did_read)) + " write=" +
str(len(did_write)) + " error=" + str(len(did_error)))
if (len(did_read) > 0):
for handle in did_read:
if (handle == sock):
(new_conn, addr) = sock.accept()
self.log(2,
"main: accept connection from " + repr(addr))
# add the accepted connection to can_read
can_read.append(new_conn)
# new_conn.send("Welcome to the LMC interface\r\n")
# an accepted connection must have generated some data
else:
try:
raw = handle.recv(4096)
except socket.error, e:
if e.errno == errno.ECONNRESET:
self.log(2, "main: closing connection")
handle.close()
for i, x in enumerate(can_read):
if (x == handle):
del can_read[i]
else:
raise e
else:
message = raw.strip()
self.log(2, "main: message='" + message + "'")
if len(message) == 0:
self.log(2, "main: closing connection")
handle.close()
for i, x in enumerate(can_read):
if (x == handle):
del can_read[i]
else:
xml = xmltodict.parse(message)
command = xml["lmc_cmd"]["command"]
if command == "reload_clients":
self.log(1, "Reloading clients")
for stream in client_streams:
self.reload_clients[stream] = True
all_reloaded = False
while (not self.parent.quit_event.isSet()
and not all_reloaded):
all_reloaded = True
for stream in client_streams:
if not self.reload_clients[stream]:
all_reloaded = False
if not all_reloaded:
self.log(
1,
"Waiting for clients to reload"
)
sleep(1)
self.log(1, "Clients reloaded")
response = "<lmc_reply>OK</lmc_reply>"
if command == "daemon_status":
response = ""
response += "<lmc_reply>"
for stream in server_streams:
response += "<stream id='" + str(
stream) + "'>"
for daemon in daemon_states[
stream].keys():
response += "<daemon name='" + daemon + "'>" + str(
daemon_states[stream]
[daemon]) + "</daemon>"
response += "</stream>"
for stream in client_streams:
response += "<stream id='" + str(
stream) + "'>"
for daemon in daemon_states[
stream].keys():
response += "<daemon name='" + daemon + "'>" + str(
daemon_states[stream]
[daemon]) + "</daemon>"
response += "</stream>"
response += "</lmc_reply>"
elif command == "host_status":
response = "<lmc_reply>"
for disk in disks.keys():
percent_full = 1.0 - (
float(disks[disk]["available"]) /
float(disks[disk]["size"]))
response += "<disk mount='" + disk + "' percent_full='" + str(
percent_full) + "'>"
response += "<size units='MB'>" + disks[
disk]["size"] + "</size>"
response += "<used units='MB'>" + disks[
disk]["used"] + "</used>"
response += "<available units='MB'>" + disks[
disk]["available"] + "</available>"
response += "</disk>"
for stream in smrbs.keys():
for key in smrbs[stream].keys():
smrb = smrbs[stream][key]
response += "<smrb stream='" + str(
stream) + "' key='" + str(
key) + "'>"
response += "<header_block nbufs='" + str(
smrb['hdr']
['nbufs']) + "'>" + str(
smrb['hdr']['full']
) + "</header_block>"
response += "<data_block nbufs='" + str(
smrb['data']['nbufs']
) + "'>" + str(smrb['data']['full']
) + "</data_block>"
response += "</smrb>"
response += "<system_load ncore='" + loads[
"ncore"] + "'>"
response += "<load1>" + loads[
"1min"] + "</load1>"
response += "<load5>" + loads[
"5min"] + "</load5>"
response += "<load15>" + loads[
"15min"] + "</load15>"
response += "</system_load>"
response += "<sensors>"
for sensor in sensors.keys():
response += "<metric name='" + sensor + "' units='" + sensors[
sensor]["units"] + "'>" + sensors[
sensor]["value"] + "</metric>"
response += "</sensors>"
response += "</lmc_reply>"
else:
response = "<lmc_reply>OK</lmc_reply>"
self.log(2, "-> " + response)
handle.send(response + "\r\n")
def main (self):
control_thread = []
# the threads for each instance of beams, streams and server
self.stream_threads = {}
self.beam_threads = {}
self.server_thread = []
self.reload_beams = {}
self.reload_streams = {}
self.system_lock = threading.Lock()
# find matching client streams for this host
self.host_streams = []
for istream in range(int(self.cfg["NUM_STREAM"])):
(req_host, beam_id, subband_id) = Config.getStreamConfig (istream, self.cfg)
if req_host == self.req_host and not istream in self.host_streams:
self.host_streams.append(istream)
# find matching client streams for this host
self.host_beams = []
for istream in range(int(self.cfg["NUM_STREAM"])):
(req_host, beam_id, subband_id) = Config.getStreamConfig (istream, self.cfg)
if req_host == self.req_host and not beam_id in self.host_beams:
self.host_beams.append(beam_id)
# find matching server stream
self.host_servers = []
if self.cfg["SERVER_HOST"] == self.req_host:
self.host_servers.append(-1)
self.server_daemon_states = {}
self.stream_daemon_states = {}
self.beam_daemon_states = {}
# configure disk systems to monitor
if len(self.host_servers) > 0:
self.disks_to_monitor = [self.cfg["SERVER_DIR"]]
else:
self.disks_to_monitor = [self.cfg["CLIENT_DIR"]]
# gather some initial statistics
self.gather_stats()
# start server thread
for stream in self.host_servers:
self.debug("server_thread["+str(stream)+"] = streamThread(-1)")
self.server_daemon_states[stream] = {}
self.server_thread = serverThread(stream, self, self.server_daemon_states[stream])
self.debug("server_thread["+str(stream)+"].start()")
self.server_thread.start()
self.debug("server_thread["+str(stream)+"] started")
sleep(5)
self.log(1, "Server threads started: " + str(len(self.host_servers)))
# start a thread for each stream
for stream in self.host_streams:
self.stream_daemon_states[stream] = {}
self.debug("stream_threads["+str(stream)+"] = streamThread ("+str(stream)+")")
self.reload_streams[stream] = False
self.stream_threads[stream] = streamThread (stream, self, self.stream_daemon_states[stream])
self.debug("stream_threads["+str(stream)+"].start()")
self.stream_threads[stream].start()
self.debug("stream_thread["+str(stream)+"] started!")
self.log(1, "Stream threads started: " + str(len(self.host_streams)))
# start a thread for each beam
for beam in self.host_beams:
self.beam_daemon_states[beam] = {}
self.debug("beam_threads["+str(beam)+"] = beamThread ("+str(beam)+")")
self.reload_beams[beam] = False
self.beam_threads[beam] = beamThread (beam, self, self.beam_daemon_states[beam])
self.debug("beam_threads["+str(beam)+"].start()")
self.beam_threads[beam].start()
self.debug("beam_thread["+str(beam)+"] started!")
self.log(1, "Beam threads started: " + str(len(self.host_beams)))
# main thread
hw_poll = 5
counter = 0
first_time = True
self.debug("starting main loop quit_event=" + str(self.quit_event.isSet()))
# control loop
while not self.quit_event.isSet():
# update the monitoring points
if counter == 0:
self.debug("refreshing monitoring points")
if not first_time:
self.resource_lock.acquire ()
self.trace("self.gather_stats()")
self.gather_stats()
first_time = False
self.resource_lock.release ()
self.trace("monitoring points refreshed")
counter = hw_poll
if not self.quit_event.isSet():
sleep(1)
counter -= 1
self.debug("quit_event set, asking lmcThreads to terminate")
self.concludeThreads()
self.debug("done")
