def preprocess(input, output, fltr=None):
header = []
header_complete = False
with codecs.open(input, 'r', 'utf-8', 'ignore') as input_file, codecs.open(output, 'wb+', 'utf-8') as output_file:
size = os.path.getsize(input)
count = 0
with Progress(size, 50, strings.converting % (os.path.basename(input), format_bytes(size))) as progress:
for line in input_file:
if fltr:
line = fltr(line)
if not line:
continue
if line.startswith('#'):
if not header_complete:
header.append(line)
else:
if line.startswith('##### CPU'): # cleanup
output_file.seek(0)
output_file.writelines(header)
fltr(None) # notify about cleanup
else:
if not header_complete:
output_file.writelines(header)
header_complete = True
output_file.write(line)
if not count % 1000:
progress.tick(input_file.tell())
count += 1
def transform_dtrace(args):
tree = default_tree()
tree['ring_buffer'] = True
TaskCombiner.disable_handling_leftovers = True
gt = get_exporters()['gt']
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
dtrace = DTrace(args, gt, callbacks)
with Progress(os.path.getsize(args.input), 50,
"Parsing: " + os.path.basename(args.input)) as progress:
count = 0
with open(args.input) as file:
for line in file:
line = line.strip()
if not line:
continue
parts = line.split('\t')
dtrace.handle_record(int(parts[0], 16), parts[1],
parts[2:])
if not count % 1000:
progress.tick(file.tell())
count += 1
dtrace.finalize()
return callbacks.get_result() + dtrace.get_result()
def join_traces(traces, output, args):
ftrace = [] # ftrace files have to be joint by time: chrome reads them in unpredictable order and complains about time
for file in traces:
if file.endswith('.ftrace') and 'merged.ftrace' != os.path.basename(file):
ftrace.append(file)
if len(ftrace) > 1: # just concatenate all files in order of creation
ftrace.sort() # name defines sorting
merged = os.path.join(os.path.dirname(ftrace[0]), 'merged.ftrace')
trace_event_clock_sync = False
with open(merged, 'w') as output_file:
for file_name in ftrace:
with open(file_name) as input_file:
for line in input_file:
if 'trace_event_clock_sync' in line:
if trace_event_clock_sync:
continue
trace_event_clock_sync = True
output_file.write(line)
traces = [file for file in traces if not file.endswith('.ftrace')]
traces.append(merged)
catapult_path = GoogleTrace.get_catapult_path(args)
if catapult_path:
import threading
proc = threading.Thread(target=GoogleTrace.catapult, args=(catapult_path, output, traces))
proc.start()
with Progress(0, 0, strings.catapulting) as progress:
count = 0
while proc.is_alive():
progress.tick(count)
count += 1
proc.join(0.5)
return output + '.html'
else:
return GoogleTrace.zip_traces(traces, output)
def transform_ftrace(args, preprocess=None):
tree = default_tree(args)
tree['ring_buffer'] = True
args.no_left_overs = True
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
with Progress(os.path.getsize(args.input), 50, "Parsing: " + os.path.basename(args.input)) as progress:
count = 0
with open(args.input) as file:
handler = FTrace(args, callbacks)
for line in file:
count += 1
if line.startswith('#'):
continue
res = FTraceImporter.parse(line)
if preprocess:
res = preprocess(res)
if not res:
continue
handler.handle_record(res['name'], res['tgid'], res['pid'], res['cpu'], res['flags'], res['time'], res['event'], res['args'])
if not count % 1000:
progress.tick(file.tell())
handler.finalize()
return callbacks.get_result()
def transform_perf(args, preprocess=None):
tree = default_tree(args)
tree['ring_buffer'] = True
args.no_left_overs = True
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
with Progress(os.path.getsize(args.input), 50,
"Parsing: " + os.path.basename(args.input)) as progress:
count = 0
with open(args.input) as file:
handler = PerfHandler(args, callbacks)
read_stack = None
for line in file:
count += 1
if not count % 1000:
progress.tick(file.tell())
line = line.strip()
if read_stack is not None:
if not line:
handler.handle_stack(read_stack)
read_stack = None
else:
read_stack.append(line)
else:
fields = parse_event(line)
handler.handle_record(fields['name'],
int(fields['pid']),
int(fields['tid']),
float(fields['time']))
if line.endswith(':'):
read_stack = []
handler.finalize()
return callbacks.get_result()
def transform_csv(args, preprocess=None):
tree = default_tree(args)
tree['ring_buffer'] = True
args.no_left_overs = True
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
with Progress(os.path.getsize(args.input), 50,
"Parsing: " + os.path.basename(args.input)) as progress:
count = 0
with open(args.input) as file:
handler = CSVHandler(args, callbacks)
header = None
for line in file:
count += 1
if not count % 1000:
progress.tick(file.tell())
if line.startswith('//'):
continue
parts = [item.strip() for item in line.strip().split(',')]
if header:
fields = dict(zip(header, parts))
if preprocess:
fields = preprocess(fields)
if not fields:
continue
handler.handle_record(fields)
else:
header = parts
handler.finalize()
return callbacks.get_result()
def transform_json(args, preprocess=None):
file_size = os.path.getsize(args.input)
if not file_size:
return []
tree = default_tree(args)
tree['ring_buffer'] = True
args.no_left_overs = True
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
with Progress(file_size, 50,
"Parsing: " + os.path.basename(args.input)) as progress:
count = 0
with codecs.open(args.input, 'r', 'utf-8',
errors='ignore') as file:
data = the_json.load(file)
handler = JsonHandler(args, callbacks)
for key, val in (enumerate(data)
if isinstance(data, list) else data.items()):
count += 1
if not count % 1000:
progress.tick(file.tell())
handler.handle_record(key, val)
handler.finalize()
return callbacks.get_result()
def zip_traces(traces, output):
import zipfile
with zipfile.ZipFile(output + ".zip", 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as zip:
count = 0
with Progress(len(traces), 50, "Merging traces") as progress:
for file in traces:
progress.tick(count)
zip.write(file, os.path.basename(file))
count += 1
return output + ".zip"
def transform_dtrace(args):
tree = default_tree(args)
tree['ring_buffer'] = True
args.no_left_overs = True
gt = get_exporters()['gt']
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
dtrace = DTrace(args, gt, callbacks)
size = os.path.getsize(args.input)
with Progress(
size, 50, "Parsing: %s (%s)" %
(os.path.basename(args.input), format_bytes(size))) as progress:
count = 0
with codecs.open(args.input, 'r', 'utf-8',
errors='ignore') as file:
reading_stack = None
stack = []
for line in file:
count += 1
ends_with_vt = (11 == ord(
line[-1])) if len(line) else False
#old_line = line
line = line.strip('\r\n')
#print "%d\t%s" % (count, line)
if not line:
if reading_stack:
dtrace.handle_stack(*(reading_stack + [stack]))
reading_stack = None
stack = []
continue
if reading_stack:
if ends_with_vt: # Vertical Tab signifies too long stack frame description
line += '...'
end_of_line = file.readline(
) # it is also treated as line end by codecs.open
line += end_of_line.strip()
stack.append(line.replace('\t', ' '))
continue
parts = line.split('\t')
if len(parts) < 4:
print("Warning: weird line:", line)
continue
if parts[1] in ['ustack', 'kstack', 'jstack']:
reading_stack = [
parts[1],
int(parts[0], 16), parts[2], parts[3].rstrip(':')
]
continue
dtrace.handle_record(int(parts[0], 16), parts[1],
parts[2:])
if not count % 1000:
progress.tick(file.tell())
dtrace.finalize()
return callbacks.get_result()
def read_db(self):
if self.db_fill:
self.db_fill = False
with self.db.select(self.table,
**{'ORDER BY': 'time'}) as data, Progress(
self.size, 50, self.table) as progress:
i = 0
for record in data:
progress.tick(i)
i += 1
name, time, args = record
fn = getattr(self.parent, name)
fn(*json.loads(args))
def read_ftrace_lines(trace, time_sync):
write_chrome_time_sync = True
with open(trace) as file:
count = 0
with Progress(os.path.getsize(trace), 50, "Loading ftrace") as progress:
for line in file:
if 'IntelSEAPI_Time_Sync' in line:
parts = line.split()
time_sync.append((float(parts[-4].strip(":")), int(parts[-1]))) # target (ftrace), source (nanosecs)
if write_chrome_time_sync: # chrome time sync, pure zero doesn't work, so we shift on very little value
yield "%strace_event_clock_sync: parent_ts=%s\n" % (line.split("IntelSEAPI_Time_Sync")[0], line.split(":")[-4].split()[-1])
write_chrome_time_sync = False # one per trace is enough
else:
yield line
if count % ProgressConst == 0:
progress.tick(file.tell())
count += 1
def transform_pprof(args):
'''
Transforms pprof trace to chosen export format.
:param args: args
:return: list of callbacks
'''
import_profile()
if not IS_AVAILABLE:
return []
tree = default_tree(args)
tree['ring_buffer'] = True
args.no_left_overs = True
prof = profile.Profile()
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
with Progress(os.path.getsize(args.input), 50,
"Parsing: " + os.path.basename(args.input)) as progress:
count = 0
with gzip.open(args.input) as input_file:
handler = PprofHandler(args, callbacks)
prof.ParseFromString(input_file.read())
handler.preprocess(prof)
time = prof.TimeNanos
pid = 0
tid = 0
for sample in prof.Sample:
duration = sample.Value[1]
if len(sample.NumLabel) != 0:
for label in sample.NumLabel:
if label.key == "pid":
pid = label.value[0]
elif label.key == "tid":
tid = label.value[0]
elif label.key == "timestamp":
time = label.value[0]
handler.handle_record("name", pid, tid, time, duration)
time += prof.Period
count += 1
if not count % 1000:
progress.tick(input_file.tell())
handler.handle_sample(sample)
handler.finalize()
return callbacks.get_result()
def transform_qnx(args):
tree = default_tree(args)
tree['ring_buffer'] = True
args.no_left_overs = True
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
parser = Parser(args, callbacks)
count = 0
with Progress(os.path.getsize(args.input), 50,
"Parsing: " + os.path.basename(args.input)) as progress:
with open(args.input) as file:
for line in file:
parser.on_line(line.strip(), count)
count += 1
if not count % 1000:
progress.tick(file.tell())
parser.finish()
return callbacks.get_result()
def transform_log(args):
from dateutil import parser
tree = default_tree(args)
tree['ring_buffer'] = True
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
log = Log(args, callbacks)
with open(args.input) as file:
with Progress(os.path.getsize(args.input), 50, "Parsing: " +
os.path.basename(args.input)) as progress:
count = 0
header = None
first_stamp = None
for line in file:
if not count % 1000:
progress.tick(file.tell())
parts = line.split()
if not header:
if len(parts) != 5:
continue
if not parts[0].startswith('Timestamp'):
print "Error: this log format is not supported. Expected log from OSX's 'log stream'"
header = parts
continue
else:
time = parser.parse(' '.join(parts[0:2]))
if first_stamp:
time = int(
(time - first_stamp).total_seconds() * 10e9)
else:
first_stamp = time
time = 0
tid = int(parts[2], 16)
type = parts[3]
activity = int(parts[4], 16)
pid = int(parts[5], 16)
msg = parts[6:]
log.handle_record(time, pid, tid, type, activity, msg)
return callbacks.get_result()
def transform_dtrace(args):
tree = default_tree(args)
tree['ring_buffer'] = True
args.no_left_overs = True
gt = get_exporters()['gt']
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
dtrace = DTrace(args, gt, callbacks)
with Progress(os.path.getsize(args.input), 50,
"Parsing: " + os.path.basename(args.input)) as progress:
count = 0
with codecs.open(args.input, 'r', 'utf-8',
errors='ignore') as file:
reading_stack = None
stack = []
for line in file:
line = line.strip()
if not line:
if reading_stack:
dtrace.handle_stack(*(reading_stack + [stack]))
reading_stack = None
stack = []
continue
if reading_stack:
stack.append(line)
continue
parts = line.split('\t')
if parts[1] == 'stack':
reading_stack = [
int(parts[0], 16), parts[2], parts[3].rstrip(':')
]
continue
dtrace.handle_record(int(parts[0], 16), parts[1],
parts[2:])
if not count % 1000:
progress.tick(file.tell())
count += 1
dtrace.finalize()
return callbacks.get_result()
def transform_ftrace(args):
tree = default_tree()
tree['ring_buffer'] = True
TaskCombiner.disable_handling_leftovers = True
with Callbacks(args, tree) as callbacks:
if callbacks.is_empty():
return callbacks.get_result()
with Progress(os.path.getsize(args.input), 50,
"Parsing: " + os.path.basename(args.input)) as progress:
count = 0
with open(args.input) as file:
handler = FTrace(args, callbacks)
for line in file:
count += 1
if line.startswith('#'):
continue
regular = line[:48].rstrip(' :')
payload = line[48:]
parts = regular.split()
if len(parts) != 4:
right = parts[-3:]
left = parts[:-3]
parts = [' '.join(left)] + right
(proc, tid) = parts[0].rsplit('-', 1)
cpu = int(parts[1][1:4])
flags = parts[2]
timestamp = float(parts[3])
(name, args) = payload.split(':', 1)
handler.handle_record(proc, int(tid), cpu,
flags, timestamp, name.strip(),
args.strip())
if not count % 1000:
progress.tick(file.tell())
handler.finalize()
return callbacks.get_result()