def convert(df, new_cdict):
# Convert the old style cdict (marshal based) to the new cdict format
# The new format is msgpack based and fixes the runtime reporting bug
# by aggregating all runtime durations into the next switch event
# and removing the runtime events
# A dict of runtime delays accumulated indexed by cpu
runtime_by_cpu = {}
# aggregate all runtimes per cpu
count = len(df)
print 'Conversion in progress...'
for index in xrange(count):
event_name = df['event'].iloc[index]
cpu = df['cpu'].iloc[index]
if event_name == 'sched__sched_stat_runtime':
try:
duration = df['duration'].iloc[index]
runtime_by_cpu[cpu] += duration
except KeyError:
# the counter is set in on the first sched switch for each cpu
pass
elif event_name == 'sched__sched_switch':
try:
if df['pid'].iloc[index]:
# don't bother to update swapper (pid=0)
runtime = runtime_by_cpu[cpu]
df.set_value(index, 'duration', runtime)
except KeyError:
pass
runtime_by_cpu[cpu] = 0
# get rid of all the runtime events
df = df[df['event'] != 'sched__sched_stat_runtime']
print 'End of conversion, marshaling and compressing...'
df.fillna(value=0, inplace=True, downcast='infer')
# save new cdict
res = {
'event': df['event'].tolist(),
'cpu': df['cpu'].tolist(),
'usecs': df['usecs'].tolist(),
'pid': df['pid'].tolist(),
'task_name': df['task_name'].tolist(),
'duration': df['duration'].tolist(),
'next_pid': df['next_pid'].tolist(),
'next_comm': df['next_comm'].tolist()
}
write_cdict(new_cdict, res)
def convert(df, new_cdict):
# Convert the old style cdict (marshal based) to the new cdict format
# The new format is msgpack based and fixes the runtime reporting bug
# by aggregating all runtime durations into the next switch event
# and removing the runtime events
# A dict of runtime delays accumulated indexed by cpu
runtime_by_cpu = {}
# aggregate all runtimes per cpu
count = len(df)
print 'Conversion in progress...'
for index in xrange(count):
event_name = df['event'].iloc[index]
cpu = df['cpu'].iloc[index]
if event_name == 'sched__sched_stat_runtime':
try:
duration = df['duration'].iloc[index]
runtime_by_cpu[cpu] += duration
except KeyError:
# the counter is set in on the first sched switch for each cpu
pass
elif event_name == 'sched__sched_switch':
try:
if df['pid'].iloc[index]:
# don't bother to update swapper (pid=0)
runtime = runtime_by_cpu[cpu]
df.set_value(index, 'duration', runtime)
except KeyError:
pass
runtime_by_cpu[cpu] = 0
# get rid of all the runtime events
df = df[df['event'] != 'sched__sched_stat_runtime']
print 'End of conversion, marshaling and compressing...'
df.fillna(value=0, inplace=True, downcast='infer')
# save new cdict
res = {'event': df['event'].tolist(),
'cpu': df['cpu'].tolist(),
'usecs': df['usecs'].tolist(),
'pid': df['pid'].tolist(),
'task_name': df['task_name'].tolist(),
'duration': df['duration'].tolist(),
'next_pid': df['next_pid'].tolist(),
'next_comm': df['next_comm'].tolist()}
write_cdict(new_cdict, res)
def capture(opts, run_name):
# If this is set we skip the capture
perf_data_filename = opts.perf_data
if perf_data_filename:
print 'Skipping capture, using ' + perf_data_filename
else:
# need to capture traces
print 'Capturing perf data for %d seconds...' % (opts.seconds)
if not perf_record(opts):
return
perf_data_filename = 'perf.data'
print 'Traces captured in perf.data'
# collect stats
if opts.all or opts.stats:
if opts.perf_data:
print 'Stats capture from provided perf data file not supported'
else:
stats_filename = opts.dest_folder + run_name + ".stats"
capture_stats(opts, stats_filename)
# create cdict from the perf data file
if opts.all or opts.switches:
try:
cdict_filename = opts.dest_folder + run_name + '.cdict'
# try to run this script through the perf tool itself as it is faster
rc = subprocess.call([perf_binary, 'script', '-s', 'mkcdict_perf_script.py', '-i', perf_data_filename])
if rc == 255:
print ' ERROR: perf is not built with the python scripting extension - aborting...'
else:
# success result is in perf.cdict, so need to rename it
os.rename('perf.cdict', cdict_filename)
os.chmod(cdict_filename, 0664)
print 'Created file: ' + cdict_filename
# remap the task names if a mapping file was provided
if opts.map:
perf_dict = perf_formatter.open_cdict(cdict_filename, opts.map)
perf_formatter.write_cdict(cdict_filename, perf_dict)
except OSError:
print 'Error: perf does not seems to be installed'
def main():
parser = OptionParser(usage="usage: %prog [options] [<run-name>]")
parser.add_option('--stats', dest='stats',
action='store_true',
default=False,
help='capture context switches and kvm exit stats to <run-name>.stats')
parser.add_option('--switches', dest='switches',
action='store_true',
default=False,
help='capture detailed context switch and kvm traces and create <run-name>.cdict if not none')
parser.add_option('--all', dest='all',
action='store_true',
default=False,
help='capture all traces and stats into <run-name>.cdict and <run-name>.stats')
parser.add_option('-s', '--sleep', dest='seconds',
action='store',
default=1,
type="int",
help='capture duration in seconds, defaults to 1 second',
metavar='<seconds>')
parser.add_option('--dest-folder', dest='dest_folder',
action='store',
default='./',
help='destination folder where to store results (default: current folder)',
metavar='<dest folder>')
parser.add_option('--use-perf-data', dest='perf_data',
action='store',
help='use given perf data file (do not capture)',
metavar='<perf data file>')
parser.add_option("--map",
dest="map",
action="store",
metavar="mapping csv file",
help="remap task names from mapping csv file"
)
parser.add_option('--use-perf', dest='perf',
action='store',
help='use given perf binary',
metavar='<perf binary>')
parser.add_option("--remap",
dest="remap",
action="store",
metavar="cdict_file",
help="remap the passed cdict file using the given map file (see --map)"
)
parser.add_option('-r', '--rc', dest='rc',
action='store',
help='source OpenStack credentials from rc file',
metavar='<openrc_file>')
parser.add_option('-p', '--password', dest='passwordd',
action='store',
help='OpenStack password',
metavar='<password>')
(opts, args) = parser.parse_args()
if len(args) > 1:
print 'This script requires 1 argument for the run name (any valid file name without extension)'
sys.exit(0)
if len(args):
run_name = args[0]
else:
run_name = 'perf'
if not os.path.isdir(opts.dest_folder):
print 'Invalid destination folder: ' + opts.dest_folder
sys.exit(2)
if not opts.dest_folder.endswith('/'):
opts.dest_folder += '/'
# check the map file is valid if provided
if opts.map:
if not os.path.exists(opts.map):
print 'ERROR: Invalid mapping file: ' + opts.map
sys.exit(1)
# pick at least one command
if opts.remap:
if not opts.map:
print 'ERROR: remap command requires a csv mapping file (--map)'
sys.exit(1)
perf_dict = perf_formatter.open_cdict(opts.remap, opts.map)
perf_formatter.write_cdict(opts.remap, perf_dict)
sys.exit(0)
if not (opts.all | opts.switches | opts.stats):
print 'Pick at least one of --stats, --switches, --all'
sys.exit(3)
CFG_FILE = '.mkcdict.cfg'
# check if there is a config file in the current directory
if os.path.isfile(CFG_FILE):
# load options from the config file if they are set to None
print 'Loading options from ' + CFG_FILE
opt_re = re.compile(' *(\w*) *[=:]+ *([\w/\-\.]*)')
with open(CFG_FILE, 'r') as ff:
for line in ff:
m = opt_re.match(line)
if m:
cfg_name = m.group(1)
try:
if getattr(opts, cfg_name) is None:
setattr(opts, cfg_name, m.group(2))
except AttributeError:
pass
if opts.perf_data and not os.path.isfile(opts.perf_data):
print 'Cannot find perf data file: ' + opts.perf_data
sys.exit(1)
if opts.perf:
global perf_binary
perf_binary = opts.perf
print 'Overriding perf binary with: ' + perf_binary
capture(opts, run_name)
if not opts.dest_folder.endswith('/'):
opts.dest_folder += '/'
# chek the map file is valid if provided
if opts.map:
if not os.path.exists(opts.map):
print 'ERROR: Invalid mapping file: ' + opts.map
sys.exit(1)
# pick at least one command
if opts.remap:
if not opts.map:
print 'ERROR: remap command requires a csv mapping file (--map)'
sys.exit(1)
perf_dict = perf_formatter.open_cdict(opts.remap, opts.map)
perf_formatter.write_cdict(opts.remap, perf_dict)
sys.exit(0)
if not (opts.all | opts.switches | opts.stats):
print 'Pick at least one of --stats, --switches, --all'
sys.exit(3)
CFG_FILE = '.mkcdict.cfg'
# check if there is a config file in the current directory
if os.path.isfile(CFG_FILE):
# load options from the config file if they are set to None
print 'Loading options from ' + CFG_FILE
opt_re = re.compile(' *(\w*) *[=:]+ *([\w/\-\.]*)')
with open(CFG_FILE, 'r') as ff:
for line in ff:
if not opts.dest_folder.endswith('/'):
opts.dest_folder += '/'
# chek the map file is valid if provided
if opts.map:
if not os.path.exists(opts.map):
print 'ERROR: Invalid mapping file: ' + opts.map
sys.exit(1)
# pick at least one command
if opts.remap:
if not opts.map:
print 'ERROR: remap command requires a csv mapping file (--map)'
sys.exit(1)
perf_dict = perf_formatter.open_cdict(opts.remap, opts.map)
perf_formatter.write_cdict(opts.remap, perf_dict)
sys.exit(0)
if not (opts.all | opts.switches | opts.stats):
print 'Pick at least one of --stats, --switches, --all'
sys.exit(3)
CFG_FILE = '.mkcdict.cfg'
# check if there is a config file in the current directory
if os.path.isfile(CFG_FILE):
# load options from the config file if they are set to None
print 'Loading options from ' + CFG_FILE
opt_re = re.compile(' *(\w*) *[=:]+ *([\w/\-\.]*)')
with open(CFG_FILE, 'r') as ff:
for line in ff:
