columns=columns2,
width=400,
height=230,
selectable=True,
editable=True)
title2 = Div(text='Task Mapping Information',
style={
'font-size': '15pt',
'color': 'black',
'text-align': 'center'
},
width=400,
height=20)
source5_df = source5.to_df()
new_source5_df = source5.to_df()
data_table2.on_change('source', lambda attr, old, new: update4())
###################################################################################################################################
points = set()
while len(points) < num_nodes:
ix = np.random.randint(1, MAX_X)
iy = np.random.randint(1, MAX_Y)
points.add((ix, iy))
x = [i[0] for i in points]
y = [i[1] for i in points]
x_label = [i - 0.3 for i in x]
y_label = [i - 0.2 for i in y]
c = brewer["Spectral"][9]
color = list(repeatlist(c, num_nodes))
assigned_task = [""] * len(points)
def main(doc=curdoc()):
print("start main function!")
class mq():
def __init__(self, outfname, subs, server, port, timeout, looptimeout):
self.OUTFNAME = outfname
self.subs = subs
self.server = server
self.port = port
self.timeout = timeout
self.looptimeout = looptimeout
self.outf = open(OUTFNAME, 'a')
self.client = mqtt.Client()
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
try:
self.client.connect(self.server, self.port, self.timeout)
except:
print("connection failed")
# The callback for when the client receives a CONNACK response from the server.
def on_connect(self, client, userdata, flags, rc):
# Subscribing in on_connect() means that if we lose the connection and
# reconnect then subscriptions will be renewed.
subres = client.subscribe(self.subs, qos=1)
print("Connected with result code " + str(rc))
# The callback for when a PUBLISH message is received from the server.
def on_message(self, client, userdata, msg):
start_messages = []
for each in tasks:
start_messages.append(each + " starts")
end_messages = []
for each in tasks:
end_messages.append(each + " ends")
# start_messages = ['localpro starts', 'aggregate0 starts', 'aggregate1 starts', 'aggregate2 starts',
# 'simpledetector0 starts', 'simpledetector1 starts', 'simpledetector2 starts', 'astutedetector0 starts',
# 'astutedetector1 starts', 'astutedetector2 starts', 'fusioncenter0 starts', 'fusioncenter1 starts',
# 'fusioncenter2 starts', 'globalfusion starts']
# end_messages = ['localpro ends', 'aggregate0 ends', 'aggregate1 ends', 'aggregate2 ends',
# 'simpledetector0 ends', 'simpledetector1 ends', 'simpledetector2 ends', 'astutedetector0 ends',
# 'astutedetector1 ends', 'astutedetector2 ends', 'fusioncenter0 ends', 'fusioncenter1 ends',
# 'fusioncenter2 ends', 'globalfusion ends']
message = msg.payload.decode()
global start_time
global finish_time
global total_time
print('--------------')
print(message)
print('--------------')
if message in start_messages:
index = start_messages.index(message)
color = "red"
doc.add_next_tick_callback(
partial(update3, index=index, color=color))
elif message in end_messages:
index = end_messages.index(message)
color1 = "blue"
doc.add_next_tick_callback(
partial(update3, index=index, color=color1))
elif message.startswith('mappings'):
print('---- Receive task mapping')
doc.add_next_tick_callback(
partial(update5,
new=message,
old=source6_df,
attr=source6.data))
doc.add_next_tick_callback(
partial(update4,
new=message,
old=source5_df,
attr=data_table2.source))
elif message.startswith('runtime'):
print('---- Receive runtime statistics')
doc.add_next_tick_callback(
partial(update8,
new=message,
old=source8_df,
attr=data_table4.source))
elif message.startswith('global'):
print('-----Receive global information')
doc.add_next_tick_callback(
partial(update7,
new=message,
old=source7_df,
attr=data_table3.source))
def retrieve_tasks(dag_info_file):
config_file = open(dag_info_file, 'r')
dag_size = int(config_file.readline())
tasks = {}
for i, line in enumerate(config_file, 1):
dag_line = line.strip().split(" ")
tasks[dag_line[0]] = i
if i == dag_size:
break
return tasks
def k8s_get_nodes(node_info_file):
"""read the node info from the file input
Args:
node_info_file (str): path of ``node.txt``
Returns:
dict: node information
"""
nodes = {}
node_file = open(node_info_file, "r")
for i, line in enumerate(node_file):
node_line = line.strip().split(" ")
nodes[i] = [node_line[0], node_line[1]]
return nodes
def repeatlist(it, count):
return islice(cycle(it), count)
def update():
m.client.loop(timeout=0.5)
# @gen.coroutine
# def update1(top,bottom,right,left,color):
# source.stream(dict(top=[top], bottom=[bottom],left=[left],right=[right],color=[color],line_color=["black"], line_width=[2]))
@gen.coroutine
def update2(x, y, time):
source1.stream(
dict(x=[x], y=[y], time=[time], text_font_style=['bold']))
@gen.coroutine
def update3(index, color):
colors = []
for i, each in enumerate(nodes):
if i == index:
colors.append(color)
continue
colors.append(each)
# print("DEBUG update3 colors:")
# print(colors)
graph.node_renderer.data_source.data['colors'] = colors
@gen.coroutine
def update4(attr, old, new):
assigned_info = new.split(' ')[1:]
for info in assigned_info:
tmp = info.split(':')
new_source5_df.loc[new_source5_df.task_names == tmp[0],
'assigned'] = tmp[1]
new_source5_df.loc[new_source5_df.task_names == tmp[0],
'as_time'] = convert_time(tmp[2])
source5.data = {
'task_id': new_source5_df.task_id,
'task_names': new_source5_df.task_names,
'assigned': new_source5_df.assigned,
'as_time': new_source5_df.as_time
}
@gen.coroutine
def update5(attr, old, new):
assigned_info = new.split(' ')[1:]
for info in assigned_info:
tmp = info.split(':')
t = '_T' + str(tasks[tmp[0]])
n = 'N' + str(node_short.index(tmp[1]))
new_source6_df.loc[new_source6_df.nodes == n,
'assigned_task'] = new_source6_df.loc[
new_source6_df.nodes == n,
'assigned_task'] + t
source6.data = {
'x': new_source6_df.x,
'y': new_source6_df.y,
'color': new_source6_df.color,
'nodes': new_source6_df.nodes,
'x_label': new_source6_df.x_label,
'y_label': new_source6_df.y_label,
'assigned_task': new_source6_df.assigned_task
}
@gen.coroutine
def update7(attr, old, new):
print("Call update7!")
tmp = new.split(' ')[1:]
home_id = tmp[0]
if tmp[1] == 'start':
data = ['N/A', 'N/A', tmp[2], tmp[0], convert_time(tmp[3])]
new_source7_df.loc[len(new_source7_df), :] = data
else:
new_source7_df.loc[(new_source7_df.home_id == tmp[0]) &
(new_source7_df.global_input == tmp[2]),
'end_times'] = convert_time(tmp[3])
tmp1 = new_source7_df.loc[(new_source7_df.home_id == tmp[0]) &
(new_source7_df.global_input == tmp[2]),
'end_times']
tmp2 = new_source7_df.loc[(new_source7_df.home_id == tmp[0]) &
(new_source7_df.global_input == tmp[2]),
'start_times']
new_source7_df.loc[(new_source7_df.home_id == tmp[0]) &
(new_source7_df.global_input == tmp[2]),
'exec_times'] = time_delta(tmp1, tmp2)
source7.data = {
'home_id': new_source7_df.home_id,
'global_input': new_source7_df.global_input,
'start_times': new_source7_df.start_times,
'end_times': new_source7_df.end_times,
'exec_times': new_source7_df.exec_times,
}
@gen.coroutine
def update8(attr, old, new):
tmp = new.split(' ')[1:]
if tmp[0] == 'enter':
data = [
'N/A', 'N/A',
convert_time(tmp[3]), 'N/A', 'N/A', tmp[2], 'N/A', tmp[1]
]
new_source8_df.loc[len(new_source8_df), :] = data
elif tmp[0] == 'exec':
new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_exec_times'] = convert_time(tmp[3])
tmp1 = new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_exec_times']
tmp2 = new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_enter_times']
new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_waiting_times'] = time_delta(tmp1, tmp2)
else:
new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_finish_times'] = convert_time(tmp[3])
tmp1 = new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_finish_times']
tmp2 = new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_enter_times']
tmp3 = new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_exec_times']
new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_elapse_times'] = time_delta(tmp1, tmp2)
new_source8_df.loc[(new_source8_df.task_name == tmp[1]) &
(new_source8_df.local_input == tmp[2]),
'local_duration_times'] = time_delta(
tmp1, tmp3)
source8.data = {
'local_duration_times': new_source8_df.local_duration_times,
'local_elapse_times': new_source8_df.local_elapse_times,
'local_enter_times': new_source8_df.local_enter_times,
'local_exec_times': new_source8_df.local_exec_times,
'local_finish_times': new_source8_df.local_finish_times,
'local_input': new_source8_df.local_input,
'local_waiting_times': new_source8_df.local_waiting_times,
'task_name': new_source8_df.task_name
}
def convert_time(t):
return datetime.datetime.fromtimestamp(
float(t)).strftime("%d.%m.%y %H:%M:%S")
def time_delta(end, start):
tmp1 = datetime.datetime.strptime(end.iloc[0], "%d.%m.%y %H:%M:%S")
tmp2 = datetime.datetime.strptime(start.iloc[0], "%d.%m.%y %H:%M:%S")
delta = (tmp1 - tmp2).total_seconds()
return delta
###################################################################################################
global OUTFNAME, SERVER_IP, SUBSCRIPTIONS, DAG_PATH, NODE_PATH
OUTFNAME = 'demo_original.html'
# SERVER_IP = 'test.mosquitto.org'
SERVER_IP = 'test.mosquitto.org'
SUBSCRIPTIONS = 'JUPITER'
DAG_PATH = 'configuration.txt'
NODE_PATH = '../../nodes.txt'
global start_time, finish_time, total_time, offset, input_num
start_time = []
finish_time = 0
total_time = 0
offset = 0
input_num = 0
global source, source1, source2, source3, source4, source5, source6, source5_df, nodes, m, p, p1
source = ColumnDataSource(data=dict(top=[0],
bottom=[0],
left=[0],
right=[0],
color=["#9ecae1"],
line_color=["black"],
line_width=[2]))
source1 = ColumnDataSource(
data=dict(x=[8], y=[3.5], time=[''], text_font_style=['bold']))
global nodes, num_nodes, MAX_X, MAX_Y, tasks
nodes = k8s_get_nodes(NODE_PATH)
num_nodes = len(nodes)
MAX_X = 10
MAX_Y = 12
tasks = retrieve_tasks(DAG_PATH)
num_tasks = len(tasks)
# doc = curdoc()
doc.title = 'CIRCE Visualization'
m = mq(outfname=OUTFNAME,
subs=SUBSCRIPTIONS,
server=SERVER_IP,
port=1883,
timeout=60,
looptimeout=1)
###################################################################################################################################
global data_table, data_table2, source5_df, new_source5_df, source6_df, new_source6_df
node_id = ['N' + str(i) for i in nodes.keys()]
node_short = [i[0] for i in nodes.values()]
node_full = [i[1] for i in nodes.values()]
source4 = ColumnDataSource(
dict(node_id=node_id, node_short=node_short, node_full=node_full))
columns = [
TableColumn(field="node_id", title="Node ID"),
TableColumn(field="node_short", title="Node Name"),
TableColumn(field="node_full", title="Full Name")
]
data_table = DataTable(source=source4,
columns=columns,
width=400,
height=230,
selectable=True)
title1 = Div(text='Node Information',
style={
'font-size': '15pt',
'color': 'black',
'text-align': 'center'
},
width=400,
height=20)
assignment = ['N/A'] * len(tasks.keys())
assign_time = ['N/A'] * len(tasks.keys())
task_id = ['T' + str(i) for i in tasks.values()]
source5 = ColumnDataSource(data=dict(task_id=task_id,
task_names=list(tasks.keys()),
assigned=assignment,
as_time=assign_time))
columns2 = [
TableColumn(field="task_id", title="Task ID"),
TableColumn(field="task_names", title="Tasks Names"),
TableColumn(field="assigned", title="Assigned Node"),
TableColumn(field="as_time", title="Assigned Time")
]
data_table2 = DataTable(source=source5,
columns=columns2,
width=400,
height=230,
selectable=True,
editable=True)
title2 = Div(text='Task Mapping Information',
style={
'font-size': '15pt',
'color': 'black',
'text-align': 'center'
},
width=400,
height=20)
source5_df = source5.to_df()
new_source5_df = source5.to_df()
data_table2.on_change('source', lambda attr, old, new: update4())
###################################################################################################################################
points = set()
while len(points) < num_nodes:
ix = np.random.randint(1, MAX_X)
iy = np.random.randint(1, MAX_Y)
points.add((ix, iy))
x = [i[0] for i in points]
y = [i[1] for i in points]
x_label = [i - 0.3 for i in x]
y_label = [i - 0.2 for i in y]
c = brewer["Spectral"][9]
color = list(repeatlist(c, num_nodes))
assigned_task = [""] * len(points)
p = figure(x_range=(0, MAX_X + 1),
y_range=(0, MAX_Y + 1),
plot_width=500,
plot_height=600)
p.background_fill_color = "#EEEDED"
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
p.xaxis.axis_label = 'Digital Ocean Clusters'
p.xaxis.axis_label_text_font_size = '20pt'
source6 = ColumnDataSource(data=dict(x=x,
y=y,
color=color,
nodes=node_id,
x_label=x_label,
y_label=y_label,
assigned_task=assigned_task))
lab = LabelSet(x='x_label',
y='y_label',
text='nodes',
text_font_style='bold',
text_color='black',
source=source6)
p.add_layout(lab)
w = p.circle(x='x',
y='y',
radius=0.8,
fill_color='color',
line_color='color',
source=source6)
p.add_tools(
HoverTool(tooltips=[
("node_id", "@nodes"),
("assigned_task", "@assigned_task"),
]))
source6_df = source6.to_df()
new_source6_df = source6.to_df()
###################################################################################################################################
global data_table3, data_table4, source7_df, new_source7_df, source8_df, new_source8_df
home_id = []
global_input = []
start_times = []
end_times = []
exec_times = []
source7 = ColumnDataSource(
dict(home_id=home_id,
global_input=global_input,
start_times=start_times,
end_times=end_times,
exec_times=exec_times))
columns = [
TableColumn(field="home_id", title="Home ID"),
TableColumn(field="global_input", title="Global Input Name"),
TableColumn(field="start_times", title="Enter time"),
TableColumn(field="end_times", title="Finish tme"),
TableColumn(field="exec_times", title="Make span [s]")
]
data_table3 = DataTable(source=source7,
columns=columns,
width=600,
height=580,
selectable=True)
title3 = Div(text='Global Input Information',
style={
'font-size': '15pt',
'color': 'black',
'text-align': 'center'
},
width=600,
height=20)
source7_df = source7.to_df()
new_source7_df = source7.to_df()
data_table3.on_change('source', lambda attr, old, new: update7())
task_name = []
local_input = []
local_enter_times = []
local_exec_times = []
local_finish_times = []
local_elapse_times = []
local_duration_times = []
local_waiting_times = []
source8 = ColumnDataSource(
dict(task_name=task_name,
local_input=local_input,
local_enter_times=local_enter_times,
local_exec_times=local_exec_times,
local_finish_times=local_finish_times,
local_elapse_times=local_elapse_times,
local_duration_times=local_duration_times,
local_waiting_times=local_waiting_times))
columns = [
TableColumn(field="task_name", title="Task name"),
TableColumn(field="local_input", title="Local Input"),
TableColumn(field="local_enter_times", title="Enter time"),
TableColumn(field="local_exec_times", title="Exec time"),
TableColumn(field="local_finish_times", title="Finish time"),
TableColumn(field="local_elapse_times", title="Elapse time"),
TableColumn(field="local_duration_times", title="Duration time"),
TableColumn(field="local_waiting_times", title="Waiting time")
]
data_table4 = DataTable(source=source8,
columns=columns,
width=900,
height=580,
selectable=True)
title4 = Div(text='Local Input Information',
style={
'font-size': '15pt',
'color': 'black',
'text-align': 'center'
},
width=600,
height=20)
source8_df = source8.to_df()
new_source8_df = source8.to_df()
data_table4.on_change('source', lambda attr, old, new: update8())
###################################################################################################################################
file = open('configuration.txt', 'r')
lines = file.readlines()
lines.pop(0)
nodes = get_dag_nodes(lines)
links = get_dag_links(lines)
print(nodes)
print(links)
G = nx.DiGraph()
G.add_nodes_from(nodes)
G.add_edges_from(links)
pos = graphviz_layout(G, prog='dot')
# calculate the range for task graph
range_x = [0, 0]
range_y = [0, 0]
for each in pos.values():
range_x[0] = min(range_x[0], int(each[0]))
range_x[1] = max(range_x[1], int(each[0]))
range_y[0] = min(range_y[0], int(each[1]))
range_y[1] = max(range_y[1], int(each[1]))
range_x = [range_x[0] - 50, range_x[1] + 150]
range_y = [range_y[0] - 50, range_y[1] + 50]
p1 = Plot(plot_width=700,
plot_height=700,
x_range=Range1d(range_x[0], range_x[1]),
y_range=Range1d(range_y[0], range_y[1]))
p1.title.text = "Network Anomaly Detection Task Graph"
p.title.text_font_size = '20pt'
node_hover_tool = HoverTool(tooltips=[("task", "@index")])
p1.add_tools(node_hover_tool, BoxZoomTool(), ResetTool())
global graph
graph = from_networkx(G, pos, scale=1, center=(0, 0))
# print("DEBUG data_source:")
# print(graph.node_renderer.data_source)
# print(graph.edge_renderer.data_source.data)
# print("DEBUG keys:")
# print(pos.keys())
# print("DEBUG values:")
# print(pos.values())
colors = []
indexs = []
for i, each in enumerate(nodes):
indexs.append(i)
colors.append('blue')
graph.node_renderer.data_source.data['colors'] = colors
graph.node_renderer.glyph = Circle(size=15, fill_color='colors')
graph.edge_renderer.glyph = MultiLine(line_color="black",
line_alpha=0.8,
line_width=1)
p1.renderers.append(graph)
# add labels to each node
x, y = zip(*graph.layout_provider.graph_layout.values())
print(x, y)
node_labels = nx.get_node_attributes(G, 'index')
source = ColumnDataSource({'x': x, 'y': y, 'task': tuple(nodes)})
labels = LabelSet(x='x',
y='y',
text='task',
source=source,
background_fill_color='white',
x_offset=(-30),
y_offset=8,
text_font_size="9pt")
p1.renderers.append(labels)
###################################################################################################################################
# node_info = column(title1,widgetbox(data_table,width=400,height=280))
# task_mapping_info = column(title2,widgetbox(data_table2,width=400,height=280))
p2 = layout([
title1,
widgetbox(data_table, width=400, height=280), title2,
widgetbox(data_table2, width=400, height=280)
],
sizing_mode='fixed',
width=400,
height=600)
p3 = layout([title3, widgetbox(data_table3)], sizing_mode='fixed')
p4 = layout([title4, widgetbox(data_table4)], sizing_mode='fixed')
doc_layout = column(row(p2, p), row(p1), row(p3, p4))
doc.add_root(doc_layout)
doc.add_periodic_callback(update, 50)
return
