def nexus_init(self):
# TODO: make this smarter
if self.last_connection > time.time() - 60:
return
for _ in range(5):
try:
nexus.init(self.config)
except paramiko.SSHException:
continue
self.last_connection = time.time()
return
abort(500)
def nexus_init(self):
# TODO: make this smarter
if self.last_connection > time.time() - 60:
return
for _ in range(5):
try:
nexus.init(self.config)
except paramiko.SSHException:
continue
self.last_connection = time.time()
return
abort(500)
def init_nexus(config):
nexus_config = {"sync_nodes": config["nodes"]["sync"], "ssh_key": SSH_KEY, "loglevel": "INFO"}
logging.info("Logging in to the sync node...")
for _ in range(5):
try:
nexus.init(nexus_config)
except paramiko.SSHException:
continue
break
else:
logging.error("Failed to reach the sync node")
exit(1)
logging.info("Logged in.")
def run(self):
for _ in range(5):
try:
nexus.init(self.config)
except paramiko.SSHException:
continue
break
else:
logging.error('Could not log in')
exit(1)
self.log.debug('Logged in.')
all_nodes = nexus.list_node_ids()
for node_id in all_nodes:
data = fetch_data(node_id, self.config)
res = assess_data(data, self.config)
node_health = {
nexus.RED: 'RED', nexus.YELLOW: 'YELLOW', nexus.GREEN: 'GREEN'
}[res.node_health]
if res.error: node_health = 'FAIL'
c = self.db.cursor()
c.execute('SELECT * FROM Cache WHERE node_id=?', [res.node_id])
for _, old_health, old_summary, old_time in c.fetchall():
if old_health == node_health: break
a = arrow.get(old_time, 'YYYY-MM-DD HH:mm:ss')
info = {
'node_id': res.node_id,
'node_health': node_health,
'summary': res.error or res.summary,
'time': arrow.utcnow().format('YYYY-MM-DD HH:mm:ss ZZ'),
'human_old_time': a.humanize(),
'old_time': a.format('YYYY-MM-DD HH:mm:ss ZZ'),
'old_health': old_health,
'old_summary': old_summary,
'config': data.config,
}
if node_health == 'FAIL':
self.log.info("{node_id} failed: {summary} "
"[was: {old_health} - {old_time}]".format(**info))
self._send_mail(info)
break
# If the status went to GREEN or got worse (* -> R || G -> Y)
if node_health == 'GREEN' or node_health == 'RED' or (
old_health == 'GREEN' and node_health == 'YELLOW'):
self.log.info("{node_id} is {node_health}: {summary} "
"[was: {old_health} - {old_time}]".format(**info))
self._send_mail(info)
self.log.debug("{} is {}: {}".format(
res.node_id,
node_health,
res.error or res.summary
))
c.execute("""INSERT OR REPLACE INTO Cache
(node_id, node_health, summary, time)
VALUES (?, ?, ?, datetime('now'));""", (
res.node_id,
node_health,
res.error or res.summary
))
self.db.commit()
def run(self):
for _ in range(5):
try:
nexus.init(self.config)
except paramiko.SSHException:
continue
break
else:
logging.error('Could not log in')
exit(1)
self.log.debug('Logged in.')
all_hubs = nexus.list_hub_ids()
for hub_id in all_hubs:
data = fetch_data(hub_id, self.config)
res = assess_data(data, self.config)
hub_health = {
nexus.RED: 'RED',
nexus.YELLOW: 'YELLOW',
nexus.GREEN: 'GREEN'
}[res.hub_health]
if res.error: hub_health = 'FAIL'
c = self.db.cursor()
c.execute('SELECT * FROM Cache WHERE hub_id=?', [res.hub_id])
for _, old_health, old_summary, old_time in c.fetchall():
if old_health == hub_health: break
a = arrow.get(old_time, 'YYYY-MM-DD HH:mm:ss')
info = {
'hub_id': res.hub_id,
'hub_health': hub_health,
'summary': res.error or res.summary,
'time': arrow.utcnow().format('YYYY-MM-DD HH:mm:ss ZZ'),
'human_old_time': a.humanize(),
'old_time': a.format('YYYY-MM-DD HH:mm:ss ZZ'),
'old_health': old_health,
'old_summary': old_summary,
'config': data.config,
}
if hub_health == 'FAIL':
self.log.info(
"{hub_id} failed: {summary} "
"[was: {old_health} - {old_time}]".format(**info))
self._send_mail(info)
break
# If the status went to GREEN or got worse (* -> R || G -> Y)
if hub_health == 'GREEN' or hub_health == 'RED' or (
old_health == 'GREEN' and hub_health == 'YELLOW'):
self.log.info(
"{hub_id} is {hub_health}: {summary} "
"[was: {old_health} - {old_time}]".format(**info))
self._send_mail(info)
self.log.debug("{} is {}: {}".format(res.hub_id, hub_health,
res.error or res.summary))
c.execute(
"""INSERT OR REPLACE INTO Cache
(hub_id, hub_health, summary, time)
VALUES (?, ?, ?, datetime('now'));""",
(res.hub_id, hub_health, res.error or res.summary))
self.db.commit()
import sqlite3
import json
import paramiko
import nexus
import sys
PARENT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(PARENT)
from monitor import get_bridge_connections
with open(os.path.expanduser('~/piloteur-config/monitor/config.json')) as f:
config = json.load(f)
for _ in range(5):
try:
nexus.init(config)
except paramiko.SSHException:
continue
break
else:
exit(1)
online_nodes = get_bridge_connections(config['bridge_host'])
DIR = os.path.dirname(os.path.abspath(__file__))
db_path = os.path.join(DIR, '..', 'cache.db')
c = sqlite3.connect(db_path).cursor()
nodes = []
c.execute('SELECT * FROM Cache')
import json
import paramiko
import os.path
from nexus import GREEN, init
from monitor import get_tunnel_connections, fetch_data, assess_data
DIR = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(DIR, 'config.json')) as f:
config = json.load(f)
for _ in range(5):
try:
init(config)
except paramiko.SSHException:
continue
break
else:
exit(1)
hubs_list = get_tunnel_connections(config['tunnel_info'])
results = []
for hub_id in hubs_list:
data = fetch_data(hub_id, config)
res = assess_data(data, config)
if res.error:
color = 'RED'
elif res.hub_health != GREEN: