def __init__(self, remote_host, control_port, vis_port):
self.remote_host = remote_host
logstream = sys.stderr
debuglevel = 1
self.log = LEDALogger(logstream, debuglevel)
self.leda = LEDARemoteHeadNodeControl(self.remote_host, control_port, self.log)
self.ledavis = LEDARemoteHeadNodeVis(self.remote_host, vis_port, self.log)
self.ledavis.open()
# self.last_vismatimage_time >= self.min_vismat_refresh_time:
"""
self.min_adcimage_refresh_time = 14
self.min_vismat_refresh_time = 10000#30
self.last_adcimage_time = 0
self.last_vismatimage_time = 0
"""
self.last_status_time = 0
self.min_refresh_time = 4
self.last_vis_update_time = 0
self.min_vis_refresh_time = 10
self.last_vis_get_time = 0
self.min_vis_get_time = 0.1
self.viserror_imgdata = open("static/images/viserror.png", "rb").read()
self.alerts = {"gpu_temp": False, "disk_use": False}
self.alert_subscribers = {"*****@*****.**": ["gpu_temp", "disk_full"]}
# Note: First is rising threshold, second is falling
self.gpu_temp_thresh = (50, 40)
self.disk_use_thresh = (90, 80)
self.updateStatus()
handlers = [(r"/", MainHandler), (r"/ajax", AJAXHandler), (r"/vis", VisHandler)]
settings = dict(
template_path=os.path.join(os.path.dirname(__file__), "templates"),
static_path=os.path.join(os.path.dirname(__file__), "static"),
xsrf_cookies=True,
autoescape="xhtml_escape",
)
tornado.web.Application.__init__(self, handlers, **settings)
class Application(tornado.web.Application):
def __init__(self, remote_host, control_port, vis_port):
self.remote_host = remote_host
logstream = sys.stderr
debuglevel = 1
self.log = LEDALogger(logstream, debuglevel)
self.leda = LEDARemoteHeadNodeControl(self.remote_host, control_port, self.log)
self.ledavis = LEDARemoteHeadNodeVis(self.remote_host, vis_port, self.log)
self.ledavis.open()
# self.last_vismatimage_time >= self.min_vismat_refresh_time:
"""
self.min_adcimage_refresh_time = 14
self.min_vismat_refresh_time = 10000#30
self.last_adcimage_time = 0
self.last_vismatimage_time = 0
"""
self.last_status_time = 0
self.min_refresh_time = 4
self.last_vis_update_time = 0
self.min_vis_refresh_time = 10
self.last_vis_get_time = 0
self.min_vis_get_time = 0.1
self.viserror_imgdata = open("static/images/viserror.png", "rb").read()
self.alerts = {"gpu_temp": False, "disk_use": False}
self.alert_subscribers = {"*****@*****.**": ["gpu_temp", "disk_full"]}
# Note: First is rising threshold, second is falling
self.gpu_temp_thresh = (50, 40)
self.disk_use_thresh = (90, 80)
self.updateStatus()
handlers = [(r"/", MainHandler), (r"/ajax", AJAXHandler), (r"/vis", VisHandler)]
settings = dict(
template_path=os.path.join(os.path.dirname(__file__), "templates"),
static_path=os.path.join(os.path.dirname(__file__), "static"),
xsrf_cookies=True,
autoescape="xhtml_escape",
)
tornado.web.Application.__init__(self, handlers, **settings)
def checkAlerts(self):
status = self.status["control"]
if "gpu_info" in status:
gpu_info = status["gpu_info"]
if "temp" in gpu_info:
gpu_temp = float(gpu_info["temp"])
self.checkAlert("gpu_temp", gpu_temp, self.gpu_temp_thresh)
if "disk_info" in status:
disk_info = status["disk_info"]
if "percent" in disk_info:
disk_use = int(disk_info["percent"])
self.checkAlert("disk_use", disk_use, self.disk_use_thresh)
def checkAlert(self, name, value, thresholds):
if not self.alerts[name]:
if value > thresholds[0]:
self.alerts[name] = True
self.raiseAlert(name, thresholds[0])
else:
if value <= thresholds[1]:
self.alerts[name] = False
def raiseAlert(self, alert_name, threshold):
recipients = []
for subsriber, subscribed_alerts in self.alert_subscribers.items():
if alert_name in subscribed_alerts:
recipients += [subscriber]
utc = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H:%M:%S")
send_email(
recipients,
"*****@*****.**",
"LEDA OVRO alert",
"The value of %s exceeded the threshold of %s at UTC %s" % (alert_name, str(threshold), utc),
)
def updateStatus(self):
print "Status request"
"""
if not self.leda.isConnected():
print "ERROR: Not connected"
return
"""
if time.time() - self.last_status_time >= self.min_refresh_time:
print "Requesting updated status from head node"
self.last_status_time = time.time()
new_status = self.leda.getStatus()
# TODO: Find a better way to deal with connections issues (isConnected/reconnect etc.)
# Handle failed control
if new_status is None:
print "Control connection down"
self.status = {}
# TODO: Set all status entries to 'error'
self.status["roach"] = []
# self.status['roach'] = [{'flow': 'ok'}]
self.status["control"] = []
self.status["headnode"] = {"host": self.remote_host, "alive": "ok", "control": "down"}
# Try to reconnect
print "Reconnecting"
self.leda.connect()
else:
self.status = new_status
self.status["headnode"] = {"host": self.remote_host, "alive": "ok", "control": "ok"}
# TODO: This needs to be re-done to check for any server/stream
# breaking (or all satisfying) the threshold conditions.
# Also need to allow simple events (e.g., corr_start)
# self.checkAlerts()
def updateVis(self):
print "Vis update request"
if time.time() - self.last_vis_update_time >= self.min_vis_refresh_time:
print "Requesting updated vis data from head node"
self.last_vis_update_time = time.time()
ret = self.ledavis.update()
if ret is None:
print "Vis connection down"
print "Reconnecting"
self.ledavis.connect()
def getVis(self, visname, i, j):
print "Request for vis '%s' (i=%i j=%i)" % (visname, i, j)
# if time.time() - self.last_vis_get_time >= self.min_vis_get_time:
imgdata = self.ledavis.get("%s=1&i=%i&j=%i" % (visname, i, j))
if imgdata is None:
print "Vis connection down, cannot get data"
imgdata = self.viserror_imgdata
# else:
# filename = "static/images/latest_vis.png"
# open(filename, 'wb').write(imgdata)
return imgdata
"""