def get_config(node_id, config, env):
init_nexus(config)
logging.info("Fetching the nodes list...")
if node_id not in nexus.list_node_ids():
logging.error("Node ID not found.")
return 1
logging.info("Node found, fetching the classes info...")
classes_log = nexus.private.fetch_system_logs("classes", node_id=node_id)
if not classes_log:
logging.error("Missing classes data.")
return 1
remote_node_id = classes_log.split(',')[0]
if not remote_node_id == node_id:
logging.error("Mismatching node_id?!")
return 1
classes = classes_log.split(',')[1:]
logging.info("Generating config...")
CONFIG_DIR = os.path.join(config['paths']['config_repo'], 'endpoint')
node_config = c.make_config(node_id, classes, config_dir=CONFIG_DIR)
print json.dumps(node_config, indent=4)
return 0
def ajax_search(self, query, refresh=False):
if not re.match(r'^[a-z0-9-\*\? ]+$', query): abort(403)
if refresh:
self.nexus_init()
all_nodes = nexus.list_node_ids()
else:
c = sqlite3.connect(self.db_path).cursor()
c.execute('SELECT node_id FROM Cache')
all_nodes = [row[0] for row in c.fetchall()]
nodes = set()
for keyword in query.split(' '):
keyword = '*' + keyword.strip('*') + '*'
nodes.update(fnmatch.filter(all_nodes, keyword))
if len(nodes) == 1:
return """
<script>window.location.href = "{}";</script>
""".format(url_for('serve_status', node_id=list(nodes)[0]))
results = []
oldest_cache = arrow.utcnow()
for node_id in nodes:
if refresh:
data = fetch_data(node_id, self.config)
res = assess_data(data, self.config)
results.append((
res.node_id,
res.node_health,
res.error or res.summary
))
continue
c.execute('SELECT * FROM Cache WHERE node_id=?', [node_id])
for node_id, node_health, summary, cache_time in c.fetchall():
cache_time = arrow.get(cache_time, 'YYYY-MM-DD HH:mm:ss')
oldest_cache = min(cache_time, oldest_cache)
node_health = {
'RED': nexus.RED, 'YELLOW': nexus.YELLOW,
'GREEN': nexus.GREEN, 'FAIL': nexus.RED
}[node_health]
results.append((node_id, node_health, summary))
if refresh or not results: oldest_cache = None
else: oldest_cache = oldest_cache.humanize()
# else: oldest_cache = '{} ({})'.format(
# oldest_cache.humanize(),
# oldest_cache.format('YYYY-MM-DD HH:mm:ss ZZ')
# )
return render_template('ajax_search.html', results=results,
nexus=nexus, oldest_cache=oldest_cache, query=query)
def test(config, env, skip_ec2 = ""):
if not skip_ec2:
### ec2.py
logging.info("Testing ec2.py...")
try:
dep_call(["./ec2.py", "--list"], config, env)
except subprocess.CalledProcessError:
logging.error("ERROR: Error executing ec2.py, check your AWS credentials")
return 1
### ssh keys
logging.info("Testing SSH keys...")
for name in ("piloteur-admin", "piloteur-devices",
"piloteur-admin.pub", "piloteur-devices.pub"):
if not os.path.exists(os.path.join(CODE, "keys", name)):
logging.error("The key %s is missing, place it in the keys/ folder", name)
return 1
logging.info("Testing github ssh connection...")
try: open_ssh("github", config).close()
except:
logging.error("ERROR: Failed to authenticate with GitHub")
return 1
### nodes
logging.info("Testing infrastructure nodes...")
for node_name in ("monitor", "bridge", "config"):
try: open_ssh(node_name, config).close()
except:
logging.error("ERROR: Failed to log into %s", node_name)
return 1
### nexus
logging.info("Testing nexus login to ...")
try:
init_nexus(config)
nexus.list_node_ids()
except:
logging.error("ERROR: Failed to make nexus work, probably sync node unreachable")
return 1
return 0
def fetch_data(node_id, c):
nexus.private.set_node_id(node_id)
if node_id not in list_node_ids():
return NodeData(node_id=node_id, error="Node ID not found.")
classes_log = nexus.private.fetch_system_logs("classes")
if not classes_log:
return NodeData(node_id=node_id, error="Missing classes data.")
remote_node_id = classes_log.split(',')[0]
if not remote_node_id == node_id:
return NodeData(node_id=node_id, error="Mismatching node_id?!")
classes = classes_log.split(',')[1:]
node_config = config.make_config(node_id, classes)
timestamp = nexus.get_timestamp()
if timestamp is None:
return NodeData(node_id=node_id, error="Missing timesync data.")
timestamp = arrow.get(timestamp)
iwconfig_log = nexus.private.fetch_system_logs("iwconfig")
if not iwconfig_log:
return NodeData(node_id=node_id, error="Missing iwconfig data.")
wifi_quality = iwconfig_log.split(',')[1]
if wifi_quality == 'N/A': wifi_quality = None
else: wifi_quality = int(wifi_quality)
versions_log = nexus.private.fetch_system_logs("versions")
if not versions_log:
return NodeData(node_id=node_id, error="Missing versions data.")
versions = versions_log.split(',')
versions = dict(zip((
"timestamp",
"ansible",
"piloteur-code",
"piloteur-blobs",
), versions))
return NodeData(node_id=node_id,
classes=classes,
config=node_config,
timestamp=timestamp,
versions=versions,
wifi_quality=wifi_quality)
def run(self):
for _ in range(5):
try:
nexus.init(self.config)
except paramiko.SSHException:
continue
break
else:
logging.error('Could not log in')
exit(1)
self.log.debug('Logged in.')
all_nodes = nexus.list_node_ids()
for node_id in all_nodes:
data = fetch_data(node_id, self.config)
res = assess_data(data, self.config)
node_health = {
nexus.RED: 'RED', nexus.YELLOW: 'YELLOW', nexus.GREEN: 'GREEN'
}[res.node_health]
if res.error: node_health = 'FAIL'
c = self.db.cursor()
c.execute('SELECT * FROM Cache WHERE node_id=?', [res.node_id])
for _, old_health, old_summary, old_time in c.fetchall():
if old_health == node_health: break
a = arrow.get(old_time, 'YYYY-MM-DD HH:mm:ss')
info = {
'node_id': res.node_id,
'node_health': node_health,
'summary': res.error or res.summary,
'time': arrow.utcnow().format('YYYY-MM-DD HH:mm:ss ZZ'),
'human_old_time': a.humanize(),
'old_time': a.format('YYYY-MM-DD HH:mm:ss ZZ'),
'old_health': old_health,
'old_summary': old_summary,
'config': data.config,
}
if node_health == 'FAIL':
self.log.info("{node_id} failed: {summary} "
"[was: {old_health} - {old_time}]".format(**info))
self._send_mail(info)
break
# If the status went to GREEN or got worse (* -> R || G -> Y)
if node_health == 'GREEN' or node_health == 'RED' or (
old_health == 'GREEN' and node_health == 'YELLOW'):
self.log.info("{node_id} is {node_health}: {summary} "
"[was: {old_health} - {old_time}]".format(**info))
self._send_mail(info)
self.log.debug("{} is {}: {}".format(
res.node_id,
node_health,
res.error or res.summary
))
c.execute("""INSERT OR REPLACE INTO Cache
(node_id, node_health, summary, time)
VALUES (?, ?, ?, datetime('now'));""", (
res.node_id,
node_health,
res.error or res.summary
))
self.db.commit()
