def convert(df, new_cdict): # Convert the old style cdict (marshal based) to the new cdict format # The new format is msgpack based and fixes the runtime reporting bug # by aggregating all runtime durations into the next switch event # and removing the runtime events # A dict of runtime delays accumulated indexed by cpu runtime_by_cpu = {} # aggregate all runtimes per cpu count = len(df) print 'Conversion in progress...' for index in xrange(count): event_name = df['event'].iloc[index] cpu = df['cpu'].iloc[index] if event_name == 'sched__sched_stat_runtime': try: duration = df['duration'].iloc[index] runtime_by_cpu[cpu] += duration except KeyError: # the counter is set in on the first sched switch for each cpu pass elif event_name == 'sched__sched_switch': try: if df['pid'].iloc[index]: # don't bother to update swapper (pid=0) runtime = runtime_by_cpu[cpu] df.set_value(index, 'duration', runtime) except KeyError: pass runtime_by_cpu[cpu] = 0 # get rid of all the runtime events df = df[df['event'] != 'sched__sched_stat_runtime'] print 'End of conversion, marshaling and compressing...' df.fillna(value=0, inplace=True, downcast='infer') # save new cdict res = { 'event': df['event'].tolist(), 'cpu': df['cpu'].tolist(), 'usecs': df['usecs'].tolist(), 'pid': df['pid'].tolist(), 'task_name': df['task_name'].tolist(), 'duration': df['duration'].tolist(), 'next_pid': df['next_pid'].tolist(), 'next_comm': df['next_comm'].tolist() } write_cdict(new_cdict, res)
def convert(df, new_cdict): # Convert the old style cdict (marshal based) to the new cdict format # The new format is msgpack based and fixes the runtime reporting bug # by aggregating all runtime durations into the next switch event # and removing the runtime events # A dict of runtime delays accumulated indexed by cpu runtime_by_cpu = {} # aggregate all runtimes per cpu count = len(df) print 'Conversion in progress...' for index in xrange(count): event_name = df['event'].iloc[index] cpu = df['cpu'].iloc[index] if event_name == 'sched__sched_stat_runtime': try: duration = df['duration'].iloc[index] runtime_by_cpu[cpu] += duration except KeyError: # the counter is set in on the first sched switch for each cpu pass elif event_name == 'sched__sched_switch': try: if df['pid'].iloc[index]: # don't bother to update swapper (pid=0) runtime = runtime_by_cpu[cpu] df.set_value(index, 'duration', runtime) except KeyError: pass runtime_by_cpu[cpu] = 0 # get rid of all the runtime events df = df[df['event'] != 'sched__sched_stat_runtime'] print 'End of conversion, marshaling and compressing...' df.fillna(value=0, inplace=True, downcast='infer') # save new cdict res = {'event': df['event'].tolist(), 'cpu': df['cpu'].tolist(), 'usecs': df['usecs'].tolist(), 'pid': df['pid'].tolist(), 'task_name': df['task_name'].tolist(), 'duration': df['duration'].tolist(), 'next_pid': df['next_pid'].tolist(), 'next_comm': df['next_comm'].tolist()} write_cdict(new_cdict, res)
def capture(opts, run_name): # If this is set we skip the capture perf_data_filename = opts.perf_data if perf_data_filename: print 'Skipping capture, using ' + perf_data_filename else: # need to capture traces print 'Capturing perf data for %d seconds...' % (opts.seconds) if not perf_record(opts): return perf_data_filename = 'perf.data' print 'Traces captured in perf.data' # collect stats if opts.all or opts.stats: if opts.perf_data: print 'Stats capture from provided perf data file not supported' else: stats_filename = opts.dest_folder + run_name + ".stats" capture_stats(opts, stats_filename) # create cdict from the perf data file if opts.all or opts.switches: try: cdict_filename = opts.dest_folder + run_name + '.cdict' # try to run this script through the perf tool itself as it is faster rc = subprocess.call([perf_binary, 'script', '-s', 'mkcdict_perf_script.py', '-i', perf_data_filename]) if rc == 255: print ' ERROR: perf is not built with the python scripting extension - aborting...' else: # success result is in perf.cdict, so need to rename it os.rename('perf.cdict', cdict_filename) os.chmod(cdict_filename, 0664) print 'Created file: ' + cdict_filename # remap the task names if a mapping file was provided if opts.map: perf_dict = perf_formatter.open_cdict(cdict_filename, opts.map) perf_formatter.write_cdict(cdict_filename, perf_dict) except OSError: print 'Error: perf does not seems to be installed'
def main(): parser = OptionParser(usage="usage: %prog [options] [<run-name>]") parser.add_option('--stats', dest='stats', action='store_true', default=False, help='capture context switches and kvm exit stats to <run-name>.stats') parser.add_option('--switches', dest='switches', action='store_true', default=False, help='capture detailed context switch and kvm traces and create <run-name>.cdict if not none') parser.add_option('--all', dest='all', action='store_true', default=False, help='capture all traces and stats into <run-name>.cdict and <run-name>.stats') parser.add_option('-s', '--sleep', dest='seconds', action='store', default=1, type="int", help='capture duration in seconds, defaults to 1 second', metavar='<seconds>') parser.add_option('--dest-folder', dest='dest_folder', action='store', default='./', help='destination folder where to store results (default: current folder)', metavar='<dest folder>') parser.add_option('--use-perf-data', dest='perf_data', action='store', help='use given perf data file (do not capture)', metavar='<perf data file>') parser.add_option("--map", dest="map", action="store", metavar="mapping csv file", help="remap task names from mapping csv file" ) parser.add_option('--use-perf', dest='perf', action='store', help='use given perf binary', metavar='<perf binary>') parser.add_option("--remap", dest="remap", action="store", metavar="cdict_file", help="remap the passed cdict file using the given map file (see --map)" ) parser.add_option('-r', '--rc', dest='rc', action='store', help='source OpenStack credentials from rc file', metavar='<openrc_file>') parser.add_option('-p', '--password', dest='passwordd', action='store', help='OpenStack password', metavar='<password>') (opts, args) = parser.parse_args() if len(args) > 1: print 'This script requires 1 argument for the run name (any valid file name without extension)' sys.exit(0) if len(args): run_name = args[0] else: run_name = 'perf' if not os.path.isdir(opts.dest_folder): print 'Invalid destination folder: ' + opts.dest_folder sys.exit(2) if not opts.dest_folder.endswith('/'): opts.dest_folder += '/' # check the map file is valid if provided if opts.map: if not os.path.exists(opts.map): print 'ERROR: Invalid mapping file: ' + opts.map sys.exit(1) # pick at least one command if opts.remap: if not opts.map: print 'ERROR: remap command requires a csv mapping file (--map)' sys.exit(1) perf_dict = perf_formatter.open_cdict(opts.remap, opts.map) perf_formatter.write_cdict(opts.remap, perf_dict) sys.exit(0) if not (opts.all | opts.switches | opts.stats): print 'Pick at least one of --stats, --switches, --all' sys.exit(3) CFG_FILE = '.mkcdict.cfg' # check if there is a config file in the current directory if os.path.isfile(CFG_FILE): # load options from the config file if they are set to None print 'Loading options from ' + CFG_FILE opt_re = re.compile(' *(\w*) *[=:]+ *([\w/\-\.]*)') with open(CFG_FILE, 'r') as ff: for line in ff: m = opt_re.match(line) if m: cfg_name = m.group(1) try: if getattr(opts, cfg_name) is None: setattr(opts, cfg_name, m.group(2)) except AttributeError: pass if opts.perf_data and not os.path.isfile(opts.perf_data): print 'Cannot find perf data file: ' + opts.perf_data sys.exit(1) if opts.perf: global perf_binary perf_binary = opts.perf print 'Overriding perf binary with: ' + perf_binary capture(opts, run_name)
if not opts.dest_folder.endswith('/'): opts.dest_folder += '/' # chek the map file is valid if provided if opts.map: if not os.path.exists(opts.map): print 'ERROR: Invalid mapping file: ' + opts.map sys.exit(1) # pick at least one command if opts.remap: if not opts.map: print 'ERROR: remap command requires a csv mapping file (--map)' sys.exit(1) perf_dict = perf_formatter.open_cdict(opts.remap, opts.map) perf_formatter.write_cdict(opts.remap, perf_dict) sys.exit(0) if not (opts.all | opts.switches | opts.stats): print 'Pick at least one of --stats, --switches, --all' sys.exit(3) CFG_FILE = '.mkcdict.cfg' # check if there is a config file in the current directory if os.path.isfile(CFG_FILE): # load options from the config file if they are set to None print 'Loading options from ' + CFG_FILE opt_re = re.compile(' *(\w*) *[=:]+ *([\w/\-\.]*)') with open(CFG_FILE, 'r') as ff: for line in ff:
if not opts.dest_folder.endswith('/'): opts.dest_folder += '/' # chek the map file is valid if provided if opts.map: if not os.path.exists(opts.map): print 'ERROR: Invalid mapping file: ' + opts.map sys.exit(1) # pick at least one command if opts.remap: if not opts.map: print 'ERROR: remap command requires a csv mapping file (--map)' sys.exit(1) perf_dict = perf_formatter.open_cdict(opts.remap, opts.map) perf_formatter.write_cdict(opts.remap, perf_dict) sys.exit(0) if not (opts.all | opts.switches | opts.stats): print 'Pick at least one of --stats, --switches, --all' sys.exit(3) CFG_FILE = '.mkcdict.cfg' # check if there is a config file in the current directory if os.path.isfile(CFG_FILE): # load options from the config file if they are set to None print 'Loading options from ' + CFG_FILE opt_re = re.compile(' *(\w*) *[=:]+ *([\w/\-\.]*)') with open(CFG_FILE, 'r') as ff: for line in ff: