def check(self): while True: # consume redis order # FIXME: should find a way to accelerate the blpop in testing task = get_redis().blpop(["sv-queue"], 1) if task is None: break else: self._do_task(task[1]) # restart dead process for p in self.processes: p.check() # post stats get_redis().set("sv-status", json.dumps(self.stats()))
def signal_process(pattern, signum): get_redis().rpush("sv-queue", json.dumps(["signal", [pattern, signum]]))
def stats_process(): return json.loads(get_redis().get("sv-status").decode())
def stop_process(pattern): get_redis().rpush("sv-queue", json.dumps(["stop", pattern]))
def start_process(cmd, wait_delay=1): get_redis().rpush("sv-queue", json.dumps(["start", [cmd, wait_delay]]))
def runner(): start_time = monotonic() start_dt = utcnow() redis = get_redis() for _, step_time in timed_loop(10, name="diagnostic"): # ram info, let's parse /proc/meminfo mems = open("/proc/meminfo").read().split("\n")[:5] mems = [l[:-3].replace(" ", "").split(":") for l in mems] mems = {k: int(v) for k, v in mems} # disk, redis, loadavg rootfs = statvfs("/") r_info = redis.info() queue_size = redis.llen("output") loadavg = open("/proc/loadavg").read().split(" ")[:3] # clock derivation runtime_dt = utcnow() - start_dt runtime = monotonic() - start_time derivation = runtime - runtime_dt.total_seconds() uptime = float(open("/proc/uptime").read().split(" ")[0]) # error counters log_name = [k.decode() for k in redis.keys("log_*")] if log_name: log_counts = [int(c) for c in redis.mget(log_name)] logs = dict(zip(log_name, log_counts)) else: logs = {} # eth1 tx/rx stats try: path = "/sys/class/net/wlan0/statistics/" target = ["tx_bytes", "tx_packets", "rx_bytes", "rx_packets"] wlan0 = {t: int(open(path + t).read()) for t in target} except: wlan0 = {} info = { "event": "board_status", "timestamp": d2s(step_time), # disk/nand status "rootfs_blocks": rootfs.f_bsize * rootfs.f_blocks, "rootfs_free": rootfs.f_bsize * rootfs.f_bfree, "rootfs_available": rootfs.f_bsize * rootfs.f_bavail, # redis info "redis_size": r_info["used_memory"], "redis_size_peak": r_info["used_memory_peak"], "redis_ops": r_info["instantaneous_ops_per_sec"], # load "loadavg1": float(loadavg[0]), "loadavg5": float(loadavg[1]), "loadavg15": float(loadavg[2]), # ram info "mem_total": mems["MemTotal"], "mem_free": mems["MemFree"], "mem_available": mems["MemAvailable"], "mem_buffers": mems["Buffers"], "mem_cached": mems["Cached"], # connection "queue_size": queue_size, # clock "derivation": derivation, "uptime": floor(uptime), } info.update(wlan0) info.update(logs) push_message("output", info)