def check(self):
while True:
# consume redis order
# FIXME: should find a way to accelerate the blpop in testing
task = get_redis().blpop(["sv-queue"], 1)
if task is None:
break
else:
self._do_task(task[1])
# restart dead process
for p in self.processes:
p.check()
# post stats
get_redis().set("sv-status", json.dumps(self.stats()))
def signal_process(pattern, signum):
get_redis().rpush("sv-queue", json.dumps(["signal", [pattern, signum]]))
def stats_process():
return json.loads(get_redis().get("sv-status").decode())
def stop_process(pattern):
get_redis().rpush("sv-queue", json.dumps(["stop", pattern]))
def start_process(cmd, wait_delay=1):
get_redis().rpush("sv-queue", json.dumps(["start", [cmd, wait_delay]]))
def runner():
start_time = monotonic()
start_dt = utcnow()
redis = get_redis()
for _, step_time in timed_loop(10, name="diagnostic"):
# ram info, let's parse /proc/meminfo
mems = open("/proc/meminfo").read().split("\n")[:5]
mems = [l[:-3].replace(" ", "").split(":") for l in mems]
mems = {k: int(v) for k, v in mems}
# disk, redis, loadavg
rootfs = statvfs("/")
r_info = redis.info()
queue_size = redis.llen("output")
loadavg = open("/proc/loadavg").read().split(" ")[:3]
# clock derivation
runtime_dt = utcnow() - start_dt
runtime = monotonic() - start_time
derivation = runtime - runtime_dt.total_seconds()
uptime = float(open("/proc/uptime").read().split(" ")[0])
# error counters
log_name = [k.decode() for k in redis.keys("log_*")]
if log_name:
log_counts = [int(c) for c in redis.mget(log_name)]
logs = dict(zip(log_name, log_counts))
else:
logs = {}
# eth1 tx/rx stats
try:
path = "/sys/class/net/wlan0/statistics/"
target = ["tx_bytes", "tx_packets", "rx_bytes", "rx_packets"]
wlan0 = {t: int(open(path + t).read()) for t in target}
except:
wlan0 = {}
info = {
"event": "board_status",
"timestamp": d2s(step_time),
# disk/nand status
"rootfs_blocks": rootfs.f_bsize * rootfs.f_blocks,
"rootfs_free": rootfs.f_bsize * rootfs.f_bfree,
"rootfs_available": rootfs.f_bsize * rootfs.f_bavail,
# redis info
"redis_size": r_info["used_memory"],
"redis_size_peak": r_info["used_memory_peak"],
"redis_ops": r_info["instantaneous_ops_per_sec"],
# load
"loadavg1": float(loadavg[0]),
"loadavg5": float(loadavg[1]),
"loadavg15": float(loadavg[2]),
# ram info
"mem_total": mems["MemTotal"],
"mem_free": mems["MemFree"],
"mem_available": mems["MemAvailable"],
"mem_buffers": mems["Buffers"],
"mem_cached": mems["Cached"],
# connection
"queue_size": queue_size,
# clock
"derivation": derivation,
"uptime": floor(uptime),
}
info.update(wlan0)
info.update(logs)
push_message("output", info)