def printKeys(trace_path) :
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
for event in trace_collection.events:
# print(event.name, " : ",event.keys(),"\n")
if re.search("net",event.name) :
print("\n", event.name, " : ")
if 'network_header' in event.keys() :
print(event['network_header'])
if 'magic' in event.keys() :
print(event['magic'])
if 'packet_size' in event.keys() :
print(event['packet_size'])
if 'skbaddr' in event.keys() :
print(event['skbaddr'])
if 'events_discarded' in event.keys() :
print(event['events_discarded'])
# print(event['content_size'])
# print(event['v'])
print("Fin de l'analyse")
def printExecveNet(trace_path) :
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
# compt1=0
# compt2=0
# compt3=0
# for event in trace_collection.events:
# if (re.search("net_dev_queue",event.name) and compt1<10):
# compt1 +=1
# print(preprocessEventsklearn(event))
# elif (re.search("execve",event.name) and compt2<10):
# compt2 += 1
# print(preprocessEventsklearn(event))
# elif (re.search("execve",event.name) and compt3<10):
# compt3 += 1
# print(preprocessEventsklearn(event))
for event in trace_collection.events:
if (re.search("net_dev_queue",event.name) and 'network_header' in event.keys()):
print("\n", event.name, " : ")
for key in event.keys() :
print("\t",key," : ",event[key])
elif (re.search("execve",event.name)and 'filename' in event.keys()):
print("\n", event.name, " : ")
for key in event.keys() :
print("\t",key," : ",event[key])
def convert(self, from_dir, to_dir):
tc = babeltrace.TraceCollection()
tc.add_trace(from_dir, "ctf")
clock = CTFWriter.Clock("clk")
writer = CTFWriter.Writer(to_dir)
writer.add_clock(clock)
stream_class = CTFWriter.StreamClass('trace')
stream_class.clock = clock
stream_class.packet_context_type.add_field(CTFWriter.IntegerFieldDeclaration(32), "cpu_id")
for id, clslst in self.handlers.items():
for cls in clslst:
for ev in cls.get_generated_events():
stream_class.add_event_class(ev)
stream = {}
for e in tc.events:
clock.time = e.timestamp
id = e["cpu_id"]
if id not in self.threads:
self.threads[id] = ThreadState(self.handlers)
stream[id] = writer.create_stream(stream_class)
stream[id].packet_context.field("cpu_id").value = id
evl = self.threads[id].consume(e)
for ev in evl:
stream[id].append_event(ev)
for s in stream.values():
s.flush()
def getEventsSynthetic(trace_path):
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
for event in trace_collection.events:
yield preprocessMoreEventsklearn(event)
def GBTPredict(trace_path):
modele = "./modeles/GBT.p"
dictVec = "./modeles/dictVec.p"
clf = joblib.load(modele)
vec = joblib.load(dictVec)
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
listeMachines = []
dicTid = {}
dictCPUid = {}
tempsDebut = datetime.now().time()
print("\tTemps debut : "+ str(tempsDebut))
for event in trace_collection.events:
try :
eventpreprocessed = preprocessMoreEventsklearn(event, listeMachines,dicTid,dictCPUid)
if eventpreprocessed != {}:
try:
if clf.predict(vec.transform(eventpreprocessed).toarray()) != [0]: # and eventpreprocessed["a_nomEvent"] != "net_dev_queue"
# print("Alerte Intrusion sur le système :")
# print(eventpreprocessed)
# print(datetime.now().time())
# print("----------------------------------")
pass
# print("Après predict")
except KeyError:
pass
except TypeError:
pass
def getExecve(trace_path) :
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
for event in trace_collection.events:
if (re.search("execve",event.name)):
yield preprocessEventsklearn(event)
def getNet(trace_path) :
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
for event in trace_collection.events:
if (re.search("net_dev_queue",event.name)):
return event
def getEventsRegleAbsolue(trace_path):
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
for event in trace_collection.events:
regles.addRegleAbsolues(event)
yield preprocessEventsklearn(event)
def generate_diagram(tracefile):
# a trace collection holds one to many traces
col = babeltrace.TraceCollection()
# add the trace provided by the user
# (LTTng traces always have the 'ctf' format)
if col.add_trace(sys.argv[1], 'ctf') is None:
raise RuntimeError('Cannot add trace')
base = None
threads = set()
switches = []
for event in col.events:
if base is None and event.name == 'atlas_job_submit':
base = event.timestamp
if base is None:
continue
if event.name == 'atlas_job_submit':
threads.add(event['tid'])
if event.name != 'sched_switch':
continue
cpu = event['cpu_id']
time = event.timestamp - base
switches.append({
'time': time,
'reason': 'D',
'tid': event['prev_tid'],
'cpu': cpu
})
switches.append({
'time': time,
'reason': 'S',
'tid': event['next_tid'],
'cpu': cpu
})
switches = [switch for switch in switches if switch['tid'] in threads]
# header
print_line("time", threads)
last = dict((thread, float('nan')) for thread in threads)
for cs in switches:
time = cs['time']
tid = cs['tid']
last[tid] = cs['cpu']
# print(time, cs)
print_line(time, [last[tid] for tid in threads])
if cs['reason'] == 'D':
last[tid] = float('nan')
print_line(time, [last[tid] for tid in threads])
def top5proc():
if len(sys.argv) != 2:
msg = 'Usage: python {} TRACEPATH'.format(sys.argv[0])
raise ValueError(msg)
# a trace collection holds one to many traces
col = babeltrace.TraceCollection()
# add the trace provided by the user
# (LTTng traces always have the 'ctf' format)
if col.add_trace(sys.argv[1], 'ctf') is None:
raise RuntimeError('Cannot add trace')
# this counter dict will hold execution times:
#
# task command name -> total execution time (ns)
exec_times = Counter()
# this holds the last `sched_switch` timestamp
last_ts = None
# iterate events
for event in col.events:
# keep only `sched_switch` events
if event.name != 'sched_switch':
continue
# keep only events which happened on CPU 0
if event['cpu_id'] != 0:
continue
# event timestamp
cur_ts = event.timestamp
if last_ts is None:
# we start here
last_ts = cur_ts
# previous task command (short) name
prev_comm = event['prev_comm']
# initialize entry in our dict if not yet done
if prev_comm not in exec_times:
exec_times[prev_comm] = 0
# compute previous command execution time
diff = cur_ts - last_ts
# update execution time of this command
exec_times[prev_comm] += diff
# update last timestamp
last_ts = cur_ts
# display top 10
for name, ns in exec_times.most_common(5):
s = ns / 1000000000
print('{:20}{} s'.format(name, s))
def babeltrace(self):
""" Gets the babeltrace trace object for the test
The test needs to have been stopped to get the trace.
"""
col = babeltrace.TraceCollection()
col.add_traces_recursive(self.lttng.output_dir, 'ctf')
return col
def comptEvent(trace_path):
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
nbEvent = 0
print(trace_path)
for event in trace_collection.events:
nbEvent += 1
print(nbEvent)
def get_trace_ctf_events(trace_directory: str) -> Iterable[BabeltraceEvent]:
"""
Get the events of a trace.
:param trace_directory: the path to the main/top trace directory
:return: events iterable
"""
tc = babeltrace.TraceCollection()
tc.add_traces_recursive(trace_directory, 'ctf')
return tc.events
def analyze_trace(path):
col = babeltrace.TraceCollection()
if col.add_trace(path, 'ctf') is None:
raise RuntimeError('Cannot add trace')
buffers = {}
for event in col.events:
if event.name.startswith("aethercast_"):
buffer_timestamp = event["timestamp"]
current_buffer = {}
if buffer_timestamp in buffers.keys():
current_buffer = buffers[buffer_timestamp]
else:
current_buffer['timestamp'] = buffer_timestamp
current_buffer[event.name] = event.timestamp
buffers[buffer_timestamp] = current_buffer
rendering_times = []
encoding_times = []
packetizing_times = []
sending_times = []
for buffer_timestamp in buffers:
buffer = buffers[buffer_timestamp]
start = buffer["aethercast_encoder:received_input_buffer"]
usec_per_sec = 1000000
renderer_end = buffer["aethercast_renderer:finished_frame"]
rendering_times.append(((renderer_end - start) / usec_per_sec))
encoder_end = buffer["aethercast_encoder:finished_frame"]
encoding_times.append(((encoder_end - start) / usec_per_sec))
packetizer_end = buffer["aethercast_packetizer:packetized_frame"]
packetizing_times.append(((packetizer_end - start) / usec_per_sec))
sender_end = buffer["aethercast_sender:sent_packet"]
sending_times.append(((sender_end - start) / usec_per_sec))
def dump_statistics(name, data):
print("%s time max: %f ms min: %f ms mean: %f ms stdev: %f ms" %
(name, max(data), min(data), statistics.mean(data),
statistics.stdev(data)))
dump_statistics("Rendering", rendering_times)
dump_statistics("Encoding", encoding_times)
dump_statistics("Packetizing", packetizing_times)
dump_statistics("Sending", sending_times)
def main():
trace_path = sys.argv[1]
print(trace_path)
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
getSomeEventsCSV(trace_path)
def load_trace(path):
"""Load the trace located in path.
Args:
path (string): Path to the LTTng trace folder.
Returns:
babeltrace.TraceCollection: a collection of one trace.
"""
trace_collection = bt.TraceCollection()
trace_collection.add_trace(path, 'ctf')
return trace_collection
def afficherDictEvents(trace_path):
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
for event in trace_collection.events:
if (re.search("net_dev_",event.name)):
print("\n", event.name, " : ")
for key in event.keys() :
print("\t",key," : ",event[key])
print("dictEvent output : ",preprocessEventsklearn(event),"\n")
print("\n\n\n-----------------------------------------------------------------------------------------------------\n\n\n")
def read(trace_path) :
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
for event in trace_collection.events:
if (re.search("syscall_entry_access",event.name) or re.search("syscall_entry_newstat",event.name) or re.search("syscall_entry_execve",event.name)) :
print("\n", event.name, " : ")
for key in event.keys() :
print("\t",key," : ",event[key])
print("Fin de l'analyse")
def generate_diagram(tracefile):
# a trace collection holds one to many traces
col = babeltrace.TraceCollection()
# add the trace provided by the user
# (LTTng traces always have the 'ctf' format)
if col.add_trace(sys.argv[1], 'ctf') is None:
raise RuntimeError('Cannot add trace')
base = None
timeline = []
active_cpus = set()
print_line("time", "cpus")
atlas_threads = set(
[e['tid'] for e in col.events if e.name == 'atlas_job_submit'])
for event in col.events:
if base is None and event.name == 'atlas_job_submit':
base = event.timestamp
print_line(0, 0)
continue
if base is None:
continue
if event.name != 'sched_switch':
continue
time = event.timestamp
cpu = event['cpu_id']
if event['prev_tid'] in atlas_threads:
# deselect
if cpu in active_cpus:
active_cpus.remove(cpu)
if event['next_tid'] in atlas_threads:
# select
active_cpus.add(event['cpu_id'])
if event['next_tid'] in atlas_threads or event[
'prev_tid'] in atlas_threads:
timeline.append({'time': time, 'num_cpus': len(active_cpus)})
timeline = [{
'time': e['time'] - base,
'num_cpus': e['num_cpus']
} for e in timeline if e['time'] > base]
for e in timeline:
print_line(e['time'], e['num_cpus'])
def test_load_trace():
traces = babeltrace.TraceCollection()
ret = traces.add_trace("nosetests/data/traceback/ust/uid/1000/64-bit/",
"ctf")
assert (ret != None)
ecount = 0
for event in traces.events:
ecount += 1
frame = event['frames'][0]
assert (type(frame.get('co_name')) == str)
assert (type(frame.get('co_filename')) == str)
assert (type(frame.get('lineno')) == int)
assert (ecount == 1)
def load_trace(paths):
if isinstance(paths, str):
paths = [paths]
trace = babeltrace.TraceCollection()
trace.size = 0
for path in paths:
if not os.path.isdir(path):
raise IOError("Path is not a directory")
for dir in find(path, "metadata"):
ret = trace.add_trace(dir, "ctf")
if ret == None:
raise IOError("failed to load trace %s" % (repr(path)))
trace.size += getFolderSize(dir)
return trace
def __init__(self, path, notifiers, stat_collector):
self.trace_collection = babeltrace.TraceCollection()
self.trace = self.trace_collection.add_traces_recursive(path, 'ctf')
self.begin_ts = self.trace_collection.timestamp_begin
self.end_ts = self.trace_collection.timestamp_end
self.stat_collector = weakref.ref(stat_collector)
state = State()
self.analysers = [
SchedAnalyser(notifiers, state),
SyscallAnalyser(notifiers, state),
IrqAnalyser(notifiers, state),
]
def readNet(trace_path) :
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
for event in trace_collection.events:
# print(event.name)
if (re.search("net_dev_queue",event.name)):
print("\n", event.name, " : ")
for key in event.keys() :
print("\t",key," : ",event[key])
print("Fin de l'analyse")
def main():
if len(sys.argv) != 3:
print("usage: ./ctf_to_catapult.py trace_directory out.json")
exit(0)
dir = sys.argv[1]
tr = bt.TraceCollection()
tr.add_trace(dir, 'ctf')
catapult_objects = list()
for event in tr.events:
if event['id'] <= 4:
catapult_objects.append(event_to_catapult(event))
catapult = {'displayTimeUnit': 'ns', 'traceEvents': catapult_objects}
with open(sys.argv[2], "w") as out:
out.write(json.dumps(catapult))
def is_trace_directory(path: str) -> bool:
"""
Check recursively if a path is a trace directory.
:param path: the path to check
:return: `True` if it is a trace directory, `False` otherwise
"""
path = os.path.expanduser(path)
if not os.path.isdir(path):
return False
tc = babeltrace.TraceCollection()
# Could still return an empty dict even if it is not a trace directory (recursively)
traces = tc.add_traces_recursive(path, 'ctf')
return traces is not None and len(traces) > 0
def check_trace_expected_timestamps(trace_paths, expected_timestamps):
traces = babeltrace.TraceCollection(intersect_mode=True)
for trace_path in trace_paths:
trace_handle = traces.add_trace(trace_path, 'ctf')
if trace_handle is None:
print('Failed to open trace at {}'.format(trace_path))
return False
for event in traces.events:
expected_timestamp = expected_timestamps.pop(0)
if event.timestamp != expected_timestamp:
print('# Unexpected timestamp ({}), expected {}'.format(
event.timestamp, expected_timestamp))
return False
return True
def main(args):
parser = argparse.ArgumentParser(description='Analyzer tool')
parser.add_argument('--path', )
parser.add_argument('--output', )
args = parser.parse_args(args)
traces = babeltrace.TraceCollection()
lttng_input = args.path
if args.output:
output_path = args.output
else:
output_path = "result.json"
ret = traces.add_trace(lttng_input, "ctf")
result = OrderedDict()
for event in traces.events:
if 'trace_id' not in event:
continue
trace_id = event['trace_id']
span_id = event['span_id']
parent_span_id = event['parent_span_id']
# init new op
if trace_id not in result:
result[trace_id] = OrderedDict()
if parent_span_id not in result[trace_id]:
result[trace_id][parent_span_id] = OrderedDict()
result[trace_id][parent_span_id][span_id] = init_zipkin_data(event)
zipkin_data = result[trace_id][parent_span_id][span_id]
result[trace_id]["start_timestamp"] = event.timestamp
result[trace_id]["latency"] = 0
if 'event' in event:
zipkin_data["events"][
event['event']] = event.timestamp - result[trace_id][
"start_timestamp"]
elif 'key' in event and 'val' in event:
zipkin_data[event['key']] = event['val']
# add this tracepoint into a leveled op
else:
result[trace_id]["latency"] = event.timestamp - result[trace_id][
"start_timestamp"]
init_zipkin_data_by_parent_span_id(
parent_span_id, span_id, result[trace_id], event,
result[trace_id]["start_timestamp"])
with open(output_path, 'w') as f:
json.dump(result, f, indent=4)
def getExecveNet(trace_path) :
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
for event in trace_collection.events:
if (re.search("net_dev_queue",event.name)):
print(preprocessMoreEventsklearn(event))
yield preprocessMoreEventsklearn(event)
elif (re.search("execve",event.name)):
print(preprocessMoreEventsklearn(event))
yield preprocessMoreEventsklearn(event)
elif (re.search("execve",event.name)):
print(preprocessMoreEventsklearn(event))
yield preprocessMoreEventsklearn(event)
def get_pruned_data(file_path, capacity):
my_deque = deque(maxlen=capacity)
trace_collection = babeltrace.TraceCollection()
if trace_collection.add_trace(file_path, 'ctf') is None:
raise RuntimeError('Cannot add trace')
for event in trace_collection.events:
temp_dict = {}
for key in event:
temp_dict[key] = event[key]
temp_dict['event_name'] = event.name
if (len(my_deque) >= capacity):
my_deque.popleft()
my_deque.append(temp_dict)
return my_deque
def readAllEvents(trace_path) :
trace_collection = babeltrace.TraceCollection()
trace_handle = trace_collection.add_trace(trace_path, 'ctf')
return trace_collection
# for event in trace_collection.events:
# if re.search("writeback",event.name) :
# print("--")
# else:
# print(event.name)
# for key in event.keys() :
# print("\t",key," : ",event[key])
print("Fin de l'analyse")
