Exemple #1
0
    def __init__(self, remote_host, control_port, vis_port):
        self.remote_host = remote_host
        logstream = sys.stderr
        debuglevel = 1
        self.log = LEDALogger(logstream, debuglevel)
        self.leda = LEDARemoteHeadNodeControl(self.remote_host, control_port, self.log)
        self.ledavis = LEDARemoteHeadNodeVis(self.remote_host, vis_port, self.log)
        self.ledavis.open()
        # self.last_vismatimage_time >= self.min_vismat_refresh_time:
        """
		self.min_adcimage_refresh_time = 14
		self.min_vismat_refresh_time = 10000#30
		self.last_adcimage_time = 0
		self.last_vismatimage_time = 0
		"""
        self.last_status_time = 0
        self.min_refresh_time = 4
        self.last_vis_update_time = 0
        self.min_vis_refresh_time = 10
        self.last_vis_get_time = 0
        self.min_vis_get_time = 0.1
        self.viserror_imgdata = open("static/images/viserror.png", "rb").read()

        self.alerts = {"gpu_temp": False, "disk_use": False}
        self.alert_subscribers = {"*****@*****.**": ["gpu_temp", "disk_full"]}
        # Note: First is rising threshold, second is falling
        self.gpu_temp_thresh = (50, 40)
        self.disk_use_thresh = (90, 80)

        self.updateStatus()
        handlers = [(r"/", MainHandler), (r"/ajax", AJAXHandler), (r"/vis", VisHandler)]
        settings = dict(
            template_path=os.path.join(os.path.dirname(__file__), "templates"),
            static_path=os.path.join(os.path.dirname(__file__), "static"),
            xsrf_cookies=True,
            autoescape="xhtml_escape",
        )
        tornado.web.Application.__init__(self, handlers, **settings)
Exemple #2
0
class Application(tornado.web.Application):
    def __init__(self, remote_host, control_port, vis_port):
        self.remote_host = remote_host
        logstream = sys.stderr
        debuglevel = 1
        self.log = LEDALogger(logstream, debuglevel)
        self.leda = LEDARemoteHeadNodeControl(self.remote_host, control_port, self.log)
        self.ledavis = LEDARemoteHeadNodeVis(self.remote_host, vis_port, self.log)
        self.ledavis.open()
        # self.last_vismatimage_time >= self.min_vismat_refresh_time:
        """
		self.min_adcimage_refresh_time = 14
		self.min_vismat_refresh_time = 10000#30
		self.last_adcimage_time = 0
		self.last_vismatimage_time = 0
		"""
        self.last_status_time = 0
        self.min_refresh_time = 4
        self.last_vis_update_time = 0
        self.min_vis_refresh_time = 10
        self.last_vis_get_time = 0
        self.min_vis_get_time = 0.1
        self.viserror_imgdata = open("static/images/viserror.png", "rb").read()

        self.alerts = {"gpu_temp": False, "disk_use": False}
        self.alert_subscribers = {"*****@*****.**": ["gpu_temp", "disk_full"]}
        # Note: First is rising threshold, second is falling
        self.gpu_temp_thresh = (50, 40)
        self.disk_use_thresh = (90, 80)

        self.updateStatus()
        handlers = [(r"/", MainHandler), (r"/ajax", AJAXHandler), (r"/vis", VisHandler)]
        settings = dict(
            template_path=os.path.join(os.path.dirname(__file__), "templates"),
            static_path=os.path.join(os.path.dirname(__file__), "static"),
            xsrf_cookies=True,
            autoescape="xhtml_escape",
        )
        tornado.web.Application.__init__(self, handlers, **settings)

    def checkAlerts(self):
        status = self.status["control"]
        if "gpu_info" in status:
            gpu_info = status["gpu_info"]
            if "temp" in gpu_info:
                gpu_temp = float(gpu_info["temp"])
                self.checkAlert("gpu_temp", gpu_temp, self.gpu_temp_thresh)
        if "disk_info" in status:
            disk_info = status["disk_info"]
            if "percent" in disk_info:
                disk_use = int(disk_info["percent"])
                self.checkAlert("disk_use", disk_use, self.disk_use_thresh)

    def checkAlert(self, name, value, thresholds):
        if not self.alerts[name]:
            if value > thresholds[0]:
                self.alerts[name] = True
                self.raiseAlert(name, thresholds[0])
        else:
            if value <= thresholds[1]:
                self.alerts[name] = False

    def raiseAlert(self, alert_name, threshold):
        recipients = []
        for subsriber, subscribed_alerts in self.alert_subscribers.items():
            if alert_name in subscribed_alerts:
                recipients += [subscriber]
        utc = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H:%M:%S")
        send_email(
            recipients,
            "*****@*****.**",
            "LEDA OVRO alert",
            "The value of %s exceeded the threshold of %s at UTC %s" % (alert_name, str(threshold), utc),
        )

    def updateStatus(self):
        print "Status request"
        """
		if not self.leda.isConnected():
			print "ERROR: Not connected"
			return
		"""
        if time.time() - self.last_status_time >= self.min_refresh_time:
            print "Requesting updated status from head node"
            self.last_status_time = time.time()
            new_status = self.leda.getStatus()

            # TODO: Find a better way to deal with connections issues (isConnected/reconnect etc.)

            # Handle failed control
            if new_status is None:
                print "Control connection down"
                self.status = {}
                # TODO: Set all status entries to 'error'
                self.status["roach"] = []
                # self.status['roach'] = [{'flow': 'ok'}]
                self.status["control"] = []
                self.status["headnode"] = {"host": self.remote_host, "alive": "ok", "control": "down"}
                # Try to reconnect
                print "Reconnecting"
                self.leda.connect()
            else:
                self.status = new_status
                self.status["headnode"] = {"host": self.remote_host, "alive": "ok", "control": "ok"}
                # TODO: This needs to be re-done to check for any server/stream
                #         breaking (or all satisfying) the threshold conditions.
                #       Also need to allow simple events (e.g., corr_start)
                # self.checkAlerts()

    def updateVis(self):
        print "Vis update request"
        if time.time() - self.last_vis_update_time >= self.min_vis_refresh_time:
            print "Requesting updated vis data from head node"
            self.last_vis_update_time = time.time()
            ret = self.ledavis.update()
            if ret is None:
                print "Vis connection down"
                print "Reconnecting"
                self.ledavis.connect()

    def getVis(self, visname, i, j):
        print "Request for vis '%s' (i=%i j=%i)" % (visname, i, j)
        # if time.time() - self.last_vis_get_time >= self.min_vis_get_time:
        imgdata = self.ledavis.get("%s=1&i=%i&j=%i" % (visname, i, j))
        if imgdata is None:
            print "Vis connection down, cannot get data"
            imgdata = self.viserror_imgdata
            # else:
            # filename = "static/images/latest_vis.png"
            # open(filename, 'wb').write(imgdata)
        return imgdata

    """