def get_recording(self, time):
"""
get the summary of the current recordings
"""
self.log_manager.pin_recording(time)
self.log_manager.get_avg_response_time()
log('NUM ANOMALIES: ', self.anomaly_cnt)
def generate_forest(self):
"""
(Deprecated) generate forest given by current executions
"""
with self.lock:
self._events2executions()
self.remove_old_data()
# self.write_file()
self._exections2forest()
# the scatterplot positions of the forest
for i, t in enumerate(self.forest[self.idx_holder["tidx"]:],
start=self.idx_holder["tidx"]):
if t['anomaly_score'] == -1 or (i %
(1000 /
(self.sampling_rate * 1000))
== 0):
self.idx_holder["tidx"] += 1
root = t['nodes'][0]
ent = root[self.layout[0]]
val = root[self.layout[1]]
rnk_thd = root[self.layout[2]] + root['threads'] * 0.1
ext = root[self.layout[3]]
self.tid.append(t["id"]) # hold tree index
self.forest_labels.append(t["anomaly_score"])
self.prog_names.append(root['prog_name'])
self.func_names.append(root['name'])
self.pos.append([ent, val, rnk_thd, ext])
log("generate {} trees".format(len(self.forest)))
self.changed = True
def get_avg_response_time(self):
ret = len(self.response_time)
if ret > 0:
avg = sum(self.response_time) / ret
log('AVG RESPONSE TIME:', avg)
return avg
else:
return ret
def set_sampling_rate():
"""
(Deprecated)
Sets sampling_rate as user adjusts.
sampling_rate is utilized to uniformly downsample the visulizing data
"""
if request.json['data'] == 'sampling_rate':
data_manager.sampling_rate = float(request.json['value'])
log("set sampling_rate #{}".format(data_manager.sampling_rate))
return jsonify({'srate': data_manager.sampling_rate})
def reset_forest(self):
"""
(Deprecated) discards forest-related variables
"""
self.pos = []
self.prog_names = []
self.func_names = []
self.forest_labels = []
self.tid = [] # hold tree index
self.eid = []
log("reset forest data")
self.changed = False
def pin_recording(self, time):
if self.is_set():
self.total_computing_time = time - self.start
log('RECEIVE TIME:', self.receive_time)
log('PUSH TIME:', self.push_time)
log('ACCUM. COMPUTING DURATION:', self.total_computing_time)
else:
log('Starting point was not set.')
def remove_old_data(self):
"""
clean executions every time_window
"""
if self.window_start // self.time_window < self.clean_count:
return
self.clean_count += 1
log("clean old executions before {}".format(self.window_start))
remove_list = []
for findex, exe in self.executions.items():
if (exe['exit'] < self.window_start):
remove_list.append(findex)
for findex in remove_list:
del self.executions[findex]
for i, t in enumerate(self.forest):
if ('nodes' in t and t['nodes'][0]['exit'] < self.window_start):
self.forest[i] = {}
def set_streamview_layout ():
'''
1) Sets number of ranks to compare for streamview
2) Sets kind of statistics for streamview
- min: minimum
- max: maximum
- mean: average
- std: standard deviation
- skn: skewness
- kts: kurtosis
- dlt: delta
'''
size = request.json['size']
stat_type = request.json['type']
print(stat_type, size)
success = False
try:
data_manager.set_stream_size(size)
data_manager.set_stream_type(stat_type)
success = True
except:
log('error occured when the layout is set.')
return jsonify({'success': success})
def _events2executionsByRank(self, rankId):
"""
(Deprecated) convert event to execution entities
"""
#print("for rank: ", rankId)
events = self.events[rankId]
#function_index = len(self.executions)
# stacks = {} #one stack for one program under the same rankId
# for i, obj in enumerate(events):
for obj in events:
self.idx_holder[rankId] += 1
# arrange event by programs first, then threads
if not obj['comm ranks'] in self.stacks:
self.stacks[obj['comm ranks']] = {}
if not obj['prog names'] in self.stacks[obj['comm ranks']]:
self.stacks[obj['comm ranks']][obj['prog names']] = {}
if not obj['threads'] in self.stacks[obj['comm ranks']][
obj['prog names']]:
self.stacks[obj['comm ranks']][obj['prog names']][
obj['threads']] = []
stack = self.stacks[obj['comm ranks']][obj['prog names']][
obj['threads']]
# check event type
if obj['event types'] == self.event_types['ENTRY']: #'entry'
#push to stack
func = {}
func['prog names'] = obj['prog names']
func['name'] = obj['name']
func['comm ranks'] = obj['comm ranks']
func['threads'] = obj['threads']
func['lineid'] = obj['lineid']
func['findex'] = self.func_idx #function_index
func['anomaly_score'] = obj['anomaly_score']
#print(func['name'], func['findex'])
if len(stack) > 0:
func['parent'] = stack[-1]['findex']
stack[-1]['children'].append(
self.func_idx) #(function_index)
#print("Children root", stack[-1]['name'], stack[-1]['entry'])
else:
func['parent'] = -1
func['children'] = []
func['entry'] = obj['timestamp']
self.func_idx += 1 #function_index+=1
stack.append(func)
elif obj['event types'] == self.event_types['EXIT']: #'exit'
if len(stack) > 0 and obj['name']:
# stack[-1]['anomaly_score'] = obj['anomaly_score']
stack[-1]['exit'] = obj['timestamp']
#self.executions.append(stack[-1])
self.executions[stack[-1]['findex']] = stack[-1]
self.idx_holder['fidx'].append(stack[-1]['findex'])
stack.pop()
else: # mismatching
log("Exit before Entry", obj['comm ranks'],
obj['prog names'], obj['threads'], obj['name'])
# print(obj)
# if len(stack) > 0:
# print("matching error "+str(i)+":"+str(rankId)+"/"+ obj['name']+"/stack: "+stack[-1]['name'])
# print([(e['name'], e['entry']) for e in stack])
# else:
# print("matching error "+str(i)+":"+str(rankId)+"/"+ obj['name']+"/empty stack")
elif obj['event types'] == self.event_types['RECV'] or obj[
'event types'] == self.event_types['SEND']:
if len(stack) > 0:
#make sure the message is correct to append
if obj['name'] != 'NA' and obj['name'] != stack[-1]['name']:
log("message issue: " + obj['name'] + ":" +
stack[-1]['name'])
#append to function
#assumption: execution never exits until message is received
if not 'messages' in stack[-1]:
stack[-1]['messages'] = []
stack[-1]['messages'].append({
"event-type":
"send" if (obj['event types']
== self.event_types['SEND']) else "receive",
"source-node-id":
obj['comm ranks'] if
(obj['event types']
== self.event_types['SEND']) else obj['partner'],
"destination-node-id":
obj['comm ranks'] if
(obj['event types']
== self.event_types['RECV']) else obj['partner'],
"thread-id":
obj['threads'], #place holder
"message-size":
obj['num bytes'],
"message-tag":
obj['Tag'],
"time":
obj['timestamp']
})
temp = stack[-1]
self.msgs.append(temp['findex'])
else:
log("Send/Recv mismatched", obj['comm ranks'],
obj['prog names'], obj['threads'], obj['name'])
# events = []
del self.events[rankId][:]
def _exections2forest(self):
"""
(Deprecated) convert executions to forest
"""
# get tree based on foi
# self.forest = []
self.lineid2treeid = {}
count = 0
while len(self.idx_holder['fidx']) > 0:
fidx = self.idx_holder['fidx'].pop(0)
if fidx in self.executions:
execution = self.executions[fidx]
if execution['name'] in self.foi:
if execution["comm ranks"] == 0: #debug
count += 1
self.lineid2treeid[execution["lineid"]] = len(self.forest)
if not "messages" in execution:
execution["messages"] = []
if (execution["exit"] - execution["entry"]) < 0:
log('negative run time detected.')
self.forest.append({
"id":
len(self.forest),
"eid":
fidx,
"prog_name":
execution["prog names"],
"node_index":
execution["comm ranks"],
"threads":
execution["threads"],
"graph_index":
len(self.forest),
"nodes": [{ # root of the tree
"name":
execution['name'], # self.foi,
"id":
0,
"comm ranks":
execution["comm ranks"],
"prog_name":
execution["prog names"],
"threads":
execution["threads"],
"findex":
execution["findex"],
"value": (execution["exit"] - execution["entry"]),
"messages":
execution["messages"],
"entry":
execution["entry"],
"exit":
execution["exit"],
"anomaly_score":
execution["anomaly_score"]
}],
"edges": [],
"anomaly_score":
execution[
'anomaly_score'] #-1 if str(int(execution["lineid"])) in self.labels else 1
})
def add_events(self, events):
"""
(Deprecated) convert events to json events
"""
with self.lock:
count = 0
prev = None
for e in events:
if self.initial_timestamp == -1: # the initial timestamp
self.initial_timestamp = int(e[11])
print("Initial time: ", self.initial_timestamp)
self.window_start = max(
0,
int(e[11]) - self.time_window - self.initial_timestamp)
obj = {
'prog names':
e[0],
'comm ranks':
e[1],
'threads':
e[2],
'event types':
e[7] if (e[3] == 'NA' or np.isnan(e[3])) else e[3],
'name':
'NA' if
(e[4] == 'NA' or np.isnan(e[4])) else self.func_dict[str(
int(e[4]))], # dictionary
'counters':
e[5],
'counter value':
e[6],
'Tag':
e[8],
'partner':
e[9],
'num bytes':
e[10],
'timestamp':
int(e[11]) - self.initial_timestamp,
'lineid':
e[12]
} # here line id is start from the beggining of the stream
count += 1
if count == len(events):
log(obj['timestamp'])
#if obj['event types'] == self.event_types['RECV'] or obj['event types'] == self.event_types['SEND']:
# print(prev)
# print(obj)
# print('\n')
if not obj['comm ranks'] in self.events:
self.events[obj['comm ranks']] = []
self.idx_holder[
obj['comm ranks']] = 0 # initialize by ranks
self.events[obj['comm ranks']].append(obj)
prev = obj
#if obj['lineid'] in self.labels:
# print(obj['lineid'], ": ", e)
if obj['event types'] == self.event_types['ENTRY']:
fname = obj["name"]
if not fname in self.stat:
self.stat[fname] = {
'abnormal': 0,
'regular': 0,
'ratio': 0
}
s = self.stat[fname]
if str(int(obj["lineid"])) in self.labels:
self.anomaly_cnt += 1
obj['anomaly_score'] = -1
s['abnormal'] = s['abnormal'] + 1
else:
obj['anomaly_score'] = 1
s['regular'] = s['regular'] + 1
if s['regular'] > 0 or s['abnormal'] > 0:
s['ratio'] = (s['abnormal'] /
(s['regular'] + s['abnormal'])) * 100
log('processed', self.anomaly_cnt, 'anomalies.')
def compute(self, data):
log('Updating Online GRA Stat ...')
log(data.items())
for rank, num in data.items():
self._compute(num)