def main(dstat_fn):
'''
First parse into CSV string, then convert to JSON
'''
with open(dstat_fn, 'r') as fid:
blob = fid.read()
fid.close()
blob = "system" + blob.split('"system"')[1].strip()
lines = blob.split('\n')
line = lines.pop(0)
cpu_list = [i.strip(' usage",,,,,') for i in line.split(',"')[1:]]
csv_str = 'time.sec,' + ','.join(cpu_list) + '\n'
line = lines.pop(0) # header row
# Parse first line and set t0
(t0, vals) = parse_line(lines.pop(0))
csv_str += '0,' + ','.join([str(val) for val in vals]) + '\n'
while lines:
(t, vals) = parse_line(lines.pop(0))
csv_str += str(round((t - t0).total_seconds(), 1)) + ','
csv_str += ','.join([str(val) for val in vals]) + '\n'
out_fn = dstat_fn.replace('data/raw', 'data/final')
out_fn += '.csv'
with open(out_fn, 'w') as fid:
fid.write(csv_str)
# Convert and save JSON object
obj = csv_to_json(csv_str)
out_fn = out_fn.replace('.csv', '.json')
with open(out_fn, 'w') as fid:
fid.write(json.dumps(obj))
def main(raw_fn):
'''
Parse each line of input file and construct CSV strings, then convert to JSON
'''
gpu_str, mem_str, pow_str = parse_raw_gpu(raw_fn)
# Often the last set of data is incomplete. Clean the csv records
gpu_str = validate(gpu_str)
mem_str = validate(mem_str)
pow_str = validate(pow_str)
ext = ['.gpu', '.mem', '.pow']
num_gpu = len(gpu_str.split('\n')[0].split(',')) - 1
header = 'time_sec,' + ','.join(['gpu' + str(i) for i in range(num_gpu)])
header += '\n'
# Save data for all individual gpu traces
for csv_str in [gpu_str, mem_str, pow_str]:
csv_str = header + csv_str
ext_str = ext.pop(0)
out_fn = raw_fn.replace('data/raw', 'data/final') + ext_str + '.csv'
with open(out_fn, 'w') as fid:
fid.write(csv_str)
obj = csv_to_json(csv_str)
out_fn = raw_fn.replace('data/raw', 'data/final') + ext_str + '.json'
with open(out_fn, 'w') as fid:
fid.write(json.dumps(obj))
# Now calculate average GPU & Memory usage for all traces and save
header = 'time_sec,GPU,MEMORY\n'
avg_str = calc_avg(gpu_str, mem_str)
out_fn = raw_fn.replace('data/raw', 'data/final') + '.avg.csv'
with open(out_fn, 'w') as fid:
fid.write(header + avg_str)
def main(raw_fn):
'''
First sum all the interrupts on a given CPU thread for a particular timestamp
Then subtract the total interrupts from previous timestamp
'''
csv_str = parse_raw_interrupts(raw_fn)
out_fn = raw_fn.replace('data/raw', 'data/final')
out_fn += '.csv'
with open(out_fn, 'w') as fid:
fid.write(csv_str)
obj = csv_to_json(csv_str)
out_fn = out_fn.replace('.csv', '.json')
with open(out_fn, 'w') as fid:
fid.write(json.dumps(obj))
def main(raw_fn):
'''
Parse each line of input file and construct CSV strings, then convert to JSON
'''
num_gpu, topology, gpu_str, membw_str, mem_str, pow_str, temp_str, pcie_str, nvl_str, int_str = parse_raw_gpu(
raw_fn)
# Often the last set of data is incomplete. Clean the csv records
# gpu_str = validate(gpu_str)
# mem_str = validate(mem_str)
# pow_str = validate(pow_str)
ext = ['.gpu', '.mem', '.pow', '.membw', '.temp']
# num_gpu = len(gpu_str.split('\n')[0].split(',')) - 1
header = 'time_sec,' + ','.join(['gpu' + str(i) for i in range(num_gpu)])
header += '\n'
# Save data for all individual gpu traces
for csv_str in [gpu_str, membw_str, mem_str, pow_str, temp_str]:
csv_str = header + csv_str
ext_str = ext.pop(0)
out_fn = raw_fn.replace('data/raw', 'data/final') + ext_str + '.csv'
with open(out_fn, 'w') as fid:
fid.write(csv_str)
obj = csv_to_json(csv_str)
out_fn = raw_fn.replace('data/raw', 'data/final') + ext_str + '.json'
with open(out_fn, 'w') as fid:
fid.write(json.dumps(obj))
# Now calculate average GPU & Memory usage for all traces and save
header = 'time_sec,GPU,MEMORY,MEM_BW\n'
avg_str = calc_avg(gpu_str, mem_str, membw_str)
out_fn = raw_fn.replace('data/raw', 'data/final') + '.avg.csv'
with open(out_fn, 'w') as fid:
fid.write(header + avg_str)
header = 'time_sec'
header += ",PCIeRx,PCIeTx,NVDevRx,NVDevTx,NVH2D,NVD2H\n"
for i in range(num_gpu):
out_fn = raw_fn.replace('data/raw', 'data/final') + '.nvl%d.csv' % (i)
with open(out_fn, 'w') as fid:
fid.write(header + int_str[i])