def finish_old_iterations():
"""
Finish the iterations that are older than 30 minutes
:return:
"""
current_f_name = inspect.currentframe().f_code.co_name
logger.info("{}: Remove_old_nodes called".format(current_f_name))
if not pipong_is_master():
return None
since = datetime.now() - timedelta(minutes=30)
s = db.session()
master_t = db.session.query(models.MasterIteration).filter(
or_(models.MasterIteration.created_date is None,
models.MasterIteration.created_date < since))
logger.debug("{}: Old iterations: {}".format(current_f_name,
master_t.count()))
for e in master_t:
e.status = "FINISHED"
s.commit()
def get_result_plot_js(master_iteration_id):
"""
Get the last result plot using html/javascript
:return:
"""
current_f_name = inspect.currentframe().f_code.co_name
if not pipong_is_master():
logger.debug(
"{}: This node is not a master}".format(
current_f_name))
abort(404)
master_it = db.session.query(models.MasterIteration).filter_by(id=master_iteration_id).first()
if master_it is None:
logger.error("{}: No MasterIteration found with id: {}".format(
current_f_name, master_iteration_id))
abort(404)
if not master_it.json_graph:
logger.error("{}: Empty json_graph for id: {}".format(
current_f_name, master_iteration_id))
abort(404)
return render_template('master/result_plot_js.html', title='Result Plot JS', graph_data_json=master_it.json_graph, master_iteration_id=master_iteration_id)
def remove_old_nodes():
"""
Delete older pinger and pongers registered in more than 30 minutes
:return:
"""
current_f_name = inspect.currentframe().f_code.co_name
logger.info("{}: Remove_old_nodes called".format(current_f_name))
if not pipong_is_master():
return None
since = datetime.now() - timedelta(minutes=30)
s = db.session()
pinger_t = db.session.query(models.RegisteredPingerNode).filter(
or_(models.RegisteredPingerNode.last_updated_date is None,
models.RegisteredPingerNode.last_updated_date < since))
logger.debug("{}: Old pingers: {}".format(current_f_name,
pinger_t.count()))
pinger_t.delete()
ponger_t = db.session.query(models.RegisteredPongerNode).filter(
or_(models.RegisteredPongerNode.last_updated_date is None,
models.RegisteredPongerNode.last_updated_date < since))
logger.debug("{}: Old pongers: {}".format(current_f_name,
ponger_t.count()))
ponger_t.delete()
s.commit()
def get_result_plot_json(master_iteration_id):
"""
Get the last result plot data in json
:return:
"""
current_f_name = inspect.currentframe().f_code.co_name
if not pipong_is_master():
logger.debug(
"{}: This node is not a master}".format(
current_f_name))
abort(404)
master_it = db.session.query(models.MasterIteration).filter_by(id=master_iteration_id).first()
if master_it is None:
logger.error("{}: No MasterIteration found with id: {}".format(
current_f_name, master_iteration_id))
abort(404)
if not master_it.json_graph:
logger.error("{}: Empty json_graph for id: {}".format(
current_f_name, master_iteration_id))
abort(404)
js_graph = json.loads(master_it.json_graph)
for e in js_graph['links']:
e['left'] = False
e['right'] = True
return jsonify(js_graph)
def get_result_plot(master_iteration_id):
"""
Get the last result plot using matplotlib
:return:
"""
current_f_name = inspect.currentframe().f_code.co_name
if not pipong_is_master():
logger.debug(
"{}: This node is not a master}".format(
current_f_name))
abort(404)
master_it = db.session.query(models.MasterIteration).filter_by(id=master_iteration_id).first()
if master_it is None:
logger.error("{}: No MasterIteration found with id: {}".format(
current_f_name, master_iteration_id))
abort(404)
if not master_it.json_graph:
logger.error("{}: Empty json_graph for id: {}".format(
current_f_name, master_iteration_id))
abort(404)
js_graph = json.loads(master_it.json_graph)
G = json_graph.node_link_graph(js_graph)
pos = nx.drawing.nx_agraph.graphviz_layout(G, prog='dot')
node_labels = {}
for k, v in G.nodes(data=True):
node_labels[k] = '{}\n{:.2f}%'.format(k, v['mean'])
nx.draw_networkx_labels(G, pos, labels=node_labels, font_size=5)
node_colors = [G.node[n]['mean'] for n in G.nodes()]
nx.draw_networkx_nodes(
G,
pos,
node_color=node_colors,
node_size=600,
node_shape='o',
cmap=plt.cm.OrRd,
vmin=0.,
vmax=100.)
nx.draw_networkx_edges(
G, pos, arrowstyle='-|>', arrowsize=20, edge_color='black', width=1)
fig = plt.gcf()
fig.set_size_inches(30, 20)
plt.savefig('/tmp/last_generated_result.png', dpi=250)
output = io.BytesIO()
FigureCanvas(fig).print_png(output)
return Response(output.getvalue(), mimetype='image/png')
def register_pinger():
"""
Register a pinger in this master node
:return:
"""
current_f_name = inspect.currentframe().f_code.co_name
if not pipong_is_master():
return jsonify({
'result': 'failure',
'msg': 'this server is not a master'
})
data = request.get_json()
ip_addr = request.remote_addr
api_port = data['api_port']
api_protocol = data['api_protocol']
registrered_t = db.session.query(models.RegisteredPingerNode).filter_by(
address=ip_addr, api_port=api_port).first()
s = db.session()
if not registrered_t:
pingp_t = models.RegisteredPingerNode(
address=ip_addr, api_port=api_port, api_protocol=api_protocol)
s.add(pingp_t)
logger.debug(
"{}: Registering ping: host:{} api_port:{} api_protocol:{}".format(
current_f_name, ip_addr, api_port, api_protocol))
else:
registrered_t.last_updated_date = datetime.now()
s.commit()
return jsonify({'result': 'success'})
def create_iteration():
"""
Create a new iteration in a master node
This will run at intervals on the master server and will
trigger the analysis of the previouly finished iteration and generate
a new master iteration with the registrered pingers
:return:
"""
current_f_name = inspect.currentframe().f_code.co_name
logger.debug("{}: Create_iteration called".format(current_f_name))
if not pipong_is_master():
return None
ponger_t = db.session.query(models.RegisteredPongerNode).all()
logger.debug("{}: Ponger_t: {}".format(current_f_name, ponger_t))
ponger_list = {}
for row in ponger_t:
ponger_list[row.address] = {
'api_port': row.api_port,
'api_protocol': row.api_protocol
}
logger.debug("{}: Ponger list: {}".format(current_f_name, ponger_list))
http_user = app.config['HTTP_AUTH_USER']
http_pass = app.config['HTTP_AUTH_PASS']
tracert_qty = app.config['MASTER_TRACERT_QTY']
s = db.session()
# get last iteration
previous_master_iter = db.session.query(models.MasterIteration).order_by(
desc(models.MasterIteration.created_date)).limit(1).first()
if previous_master_iter:
logger.debug("{}: Previous_master_iter: {} status:{}".format(
current_f_name, previous_master_iter.id,
previous_master_iter.status))
if previous_master_iter.status != 'FINISHED':
logger.error(
"{}: Cannot start a new iteration while the previous one (id:{}) is not FINISHED (status:{})"
.format(current_f_name, previous_master_iter.id,
previous_master_iter.status))
return None
# create a new master iteration
master_ite_t = models.MasterIteration()
s.add(master_ite_t)
s.flush()
s.commit()
# start the pinger sessions
pinger_t = db.session.query(models.RegisteredPingerNode).all()
for pinger in pinger_t:
plist = dict(ponger_list)
if pinger.address in plist.keys():
del plist[pinger.address]
if len(plist.keys()) > 0:
s.add(
models.MasterIterationPinger(
master_iteration_id=master_ite_t.id,
registered_pinger_id=pinger.id,
status="RUNNING"))
s.commit()
else:
# dont call any pinger that does not have pongers to query
continue
post_url = "http://{}:{}/api/v1.0/start_session".format(
pinger.address, pinger.api_port)
post_json = {
"hosts": plist,
"tracert_qty": tracert_qty,
"master_iteration_id": master_ite_t.id
}
try:
logger.debug("post url: {} json:{}".format(post_url, post_json))
requests.post(post_url,
auth=requestHTTPAuth(http_user, http_pass),
json=post_json,
timeout=5)
except Exception as e:
logger.error(
"{}: Error calling create session on pinger {}:{} {}".format(
current_f_name, pinger.address, pinger.api_port, str(e)))
logger.debug("{}: Create_iteration finished".format(current_f_name))
def test_if_master(self):
"""
Check if instance is master
:return:
"""
assert pipong_is_master() is True
def home():
"""
index view
return the status of this node:
if ponger returns the pinger registrered in this node
if pinger returns the list of pongers and last iteration status
if master returns list of pongers and pingers and last iteration result
:return:
"""
current_f_name = inspect.currentframe().f_code.co_name
#logger.info('{}: home called'.format(current_f_name))
result_dict = {}
is_pinger = pipong_is_pinger()
is_ponger = pipong_is_ponger()
is_master = pipong_is_master()
result_dict['capabilities'] = {
'is_pinger': is_pinger,
'is_ponger': is_ponger,
'is_master': is_master
}
if is_pinger:
last_iteration_status = {}
ponger_list = []
iteration = db.session.query(models.PingerIteration).order_by(
desc(models.PingerIteration.created_date)).limit(1).first()
if iteration:
last_iteration_status = {
'id': iteration.id,
'remote_address': iteration.remote_address,
'status': iteration.status,
'tracert_qty': iteration.tracert_qty,
'created_date': iteration.created_date
}
for p in iteration.ponger:
ponger_list.append({
'address': p.address,
'api_port': p.api_port,
'api_protocol': p.api_protocol
})
result_dict['pinger_info'] = {
'last_iteration_status': last_iteration_status,
'ponger_list': ponger_list
}
if is_ponger:
pinger_port_list = []
pinger_port_t = db.session.query(models.AllocatedPingerPort)
for p in pinger_port_t:
pinger_port_list.append({'address': p.address, 'port': p.port})
result_dict['ponger_info'] = {'pinger_port_list': pinger_port_list}
if is_master:
registrered_pingers = []
registrered_pongers = []
result_dict['master_info'] = {}
i = 0
iter_ids = [
e.id for e in db.session.query(models.MasterIteration).order_by(
desc(models.MasterIteration.created_date)).limit(2).all()
]
for m_iter_id in iter_ids:
m_iter = db.session.query(
models.MasterIteration).filter_by(id=m_iter_id).first()
it_id = int(m_iter.id)
it_info = {
'id':
int(m_iter.id),
'status':
str(m_iter.status),
'has_graph':
True if m_iter.json_graph else False,
'created_date':
str(m_iter.created_date),
'problematic_hosts': [{
'host': str(h.problematic_host),
'score': float(h.score)
} for h in m_iter.master_iteration_result]
}
iteration_label = "previous_iteration"
if i == 0:
iteration_label = "current_iteration"
res = tasks.master_tasks.check_master_iteration_done.apply(
args=[it_id])
it_info['progress'] = res.get()
result_dict['master_info'][iteration_label] = it_info
i += 1
r_pingers_t = db.session.query(models.RegisteredPingerNode)
for pinger in r_pingers_t:
registrered_pingers.append({
'address': pinger.address,
'api_port': pinger.api_port,
'api_protocol': pinger.api_protocol
})
r_pongers_t = db.session.query(models.RegisteredPongerNode)
for ponger in r_pongers_t:
registrered_pongers.append({
'address': ponger.address,
'api_port': ponger.api_port,
'api_protocol': ponger.api_protocol
})
result_dict['master_info']['registrered_pingers'] = registrered_pingers
result_dict['master_info']['registrered_pongers'] = registrered_pongers
return jsonify(result_dict)
def register_pinger_result():
"""
Register a pinger results in this master node
The results came in JSON format and are related to the iperf
session performed on the pinger node
:return:
"""
current_f_name = inspect.currentframe().f_code.co_name
logger.info("{}: called".format(current_f_name))
if not pipong_is_master():
logger.info("{}: pipong_is_master:{}".format(current_f_name,
pipong_is_master()))
return jsonify({
'result': 'failure',
'msg': 'this server is not a master'
})
data = request.get_json()
ip_addr = request.remote_addr
master_iteration_id = data['master_remote_id']
pinger_port = data['local_port']
pinger_result = data['result']
registrered_t = db.session.query(models.RegisteredPingerNode).filter_by(
address=ip_addr, api_port=pinger_port).first()
if not registrered_t:
logger.error(
"{}: Error, the pinger node was not registered {}:{}".format(
current_f_name, ip_addr, pinger_port))
return jsonify({
'result': 'failure',
'msg': 'the pinger node was not registered'
})
pinger_iteration_t = db.session.query(
models.MasterIterationPinger).filter_by(
master_iteration_id=master_iteration_id,
registered_pinger_id=registrered_t.id).first()
if not pinger_iteration_t:
logger.error("{}: Error, the master pinger iteration was not found. "
"Master iter:{} registered pinger:{}".format(
current_f_name, master_iteration_id,
registrered_t.id))
return jsonify({
'result': 'failure',
'msg': 'the master pinger iteration was not found'
})
if pinger_iteration_t.status == "FINISHED":
logger.error("{}: Error, the pinger iteration was finished. "
"Pinger iteration:{} status:{}".format(
current_f_name, pinger_iteration_t.id,
pinger_iteration_t.status))
return jsonify({
'result':
'failure',
'msg':
' the master pinger iteration is already finished'
})
s = db.session()
for e in pinger_result:
e['pinger_address'] = ip_addr
pinger_iteration_t.result = str(pinger_result)
pinger_iteration_t.status = "FINISHED"
s.commit()
logger.info(
"{}: Pinger result registrered. Pinger address:{} result: {}".format(
current_f_name, ip_addr, str(pinger_result)))
res = tasks.master_tasks.check_master_iteration_done(master_iteration_id)
logger.debug(
"{}: check_master_iteration_done: {}".format(
current_f_name, res))
if res['is_finished']:
# big info message on the logs for easy visualization
logger.info("{}: ################################".format(current_f_name))
logger.info("{}: # ITERATION id:{} FINISHED".format(current_f_name, master_iteration_id))
logger.info("{}: ################################".format(current_f_name))
# analyse last iteration results
tasks.master_tasks.analyse_iteration.apply_async(args=[master_iteration_id], kwargs={})
return jsonify({'result': 'success'})
