def plotIpt(directory): # Read inter-packet arrival times from the sniffer pickled files ipt_pfile = os.path.join(directory, "pickled/ipt.txt") ipt_summary_pfile = os.path.join(directory, "pickled/ipt_summary.txt") ipt = readPickledFile(ipt_pfile) summary = readPickledFile(ipt_summary_pfile) ports = summary.keys() # Read packet lengths from sniffer pickled files pkt_len_freq_pfile = os.path.join(directory, "pickled/pkt_len_freq.txt") (most_freq_pkt_len, pkt_len_freq) = readPickledFile(pkt_len_freq_pfile) cdf_data = [] avg_data = [] pc99_data = [] for port in ports: # Convert from nanoseconds to microseconds for plotting cdf_data.append(map(lambda x: x / 1000.0, ipt[port])) avg, pc99 = summary[port] avg_data.append(avg / 1000.0) pc99_data.append(pc99 / 1000.0) # Generate CDF plot plot = plotCDFGraphSimple( cdf_data, avg_data, pc99_data, "Inter-packet time (in microseconds)", "Fractiles", "CDF of inter-packet time (not inter-packet gap) " "in microseconds", ) # Compute ideal inter-packet arrival time for most frequently seen packet # length in the trace rate_mbps = float(readDirTagFileProperty(directory, "rate_mbps")) nclasses = int(readDirTagFileProperty(directory, "nclasses")) rate_per_class_gbps = rate_mbps / (1000.0 * nclasses) ideal_nsec = idealIptNsec(most_freq_pkt_len, rate_per_class_gbps) # Add VLine for the ideal inter-packet time # Also convert from nanoseconds to microseconds for plotting ideal_vline = boomslang.VLine(color="magenta", lineStyle="-.", width=2) ideal_vline.xValues = [ideal_nsec / 1000.0] ideal_vline.label = "Ideal" plot.add(ideal_vline) return plot
def plotIpt(directory): # Read inter-packet arrival times from the sniffer pickled files ipt_pfile = os.path.join(directory, 'pickled/ipt.txt') ipt_summary_pfile = os.path.join(directory, 'pickled/ipt_summary.txt') ipt = readPickledFile(ipt_pfile) summary = readPickledFile(ipt_summary_pfile) ports = summary.keys() # Read packet lengths from sniffer pickled files pkt_len_freq_pfile = os.path.join(directory, 'pickled/pkt_len_freq.txt') (most_freq_pkt_len, pkt_len_freq) = readPickledFile(pkt_len_freq_pfile) cdf_data = [] avg_data = [] pc99_data = [] for port in ports: # Convert from nanoseconds to microseconds for plotting cdf_data.append(map(lambda x: x / 1000.0, ipt[port])) avg, pc99 = summary[port] avg_data.append(avg / 1000.0) pc99_data.append(pc99 / 1000.0) # Generate CDF plot plot = plotCDFGraphSimple( cdf_data, avg_data, pc99_data, "Inter-packet time (in microseconds)", "Fractiles", "CDF of inter-packet time (not inter-packet gap) " "in microseconds") # Compute ideal inter-packet arrival time for most frequently seen packet # length in the trace rate_mbps = float(readDirTagFileProperty(directory, "rate_mbps")) nclasses = int(readDirTagFileProperty(directory, "nclasses")) rate_per_class_gbps = rate_mbps / (1000.0 * nclasses) ideal_nsec = idealIptNsec(most_freq_pkt_len, rate_per_class_gbps) # Add VLine for the ideal inter-packet time # Also convert from nanoseconds to microseconds for plotting ideal_vline = boomslang.VLine(color="magenta", lineStyle="-.", width=2) ideal_vline.xValues = [ideal_nsec / 1000.0] ideal_vline.label = "Ideal" plot.add(ideal_vline) return plot
def main(argv): # Parse flags args = parser.parse_args() # Check if the base directory is valid exists = os.path.exists(args.base_dir) isdir = os.path.isdir(args.base_dir) if not exists: print 'Base directory does not exist' sys.exit(1) if not isdir: print 'Base directory not valid' sys.exit(1) args.base_dir = os.path.abspath(args.base_dir) # Create Plumbum commands for pickling data and plotting pickle_cmd = plumbum.local[os.path.join(config['PLOT_SCRIPTS_DIR'], 'pickleExptLogs.py')] plot_sniffer_cmd = plumbum.local[os.path.join(config['PLOT_SCRIPTS_DIR'], 'plotSniffer.py')] plot_mcperf_lat_cmd = plumbum.local[os.path.join( config['PLOT_SCRIPTS_DIR'], 'plotMcperfLatency.py')] # Temp directory for plotting graphs plot_tmpdir = config['PLOT_TMPDIR'] print 'Exploring base directory :', args.base_dir num_exp_dirs = 0 for (path, dirs, files) in os.walk(args.base_dir, followlinks=True): # Check if an experiment directory was found if os.path.exists(os.path.join(path, 'expsift_tags')): print 'Found experiment directory:', path num_exp_dirs += 1 # Pickle experiment logs print '... Pickling experiment data' if args.force_replot: pickle_cmd('-f', path, plot_tmpdir) else: pickle_cmd(path, plot_tmpdir) # Read the workload type for the experiment directory workload = readDirTagFileProperty(path, 'workload') if (not workload in trafgen_workloads and not workload in memcached_workloads): print 'Workload not recognized for expt: %s' % args.expt_dir sys.exit(1) # Plot experiment graphs print '... Plotting experiment graphs' expt_plot_dir = os.path.join(path, 'plot/') if not os.path.exists(expt_plot_dir): os.makedirs(expt_plot_dir) if workload in trafgen_workloads: if args.force_replot: plot_sniffer_cmd('-f', path, expt_plot_dir) else: plot_sniffer_cmd(path, expt_plot_dir) elif workload in memcached_workloads: if args.force_replot: plot_mcperf_lat_cmd('-f', path, expt_plot_dir) else: plot_mcperf_lat_cmd(path, expt_plot_dir) print 'Plotted graphs for %d experiments under %s' % (num_exp_dirs, args.base_dir)
def main(argv): # Parse flags args = parser.parse_args() # Read the workload type for the experiment directory workload = readDirTagFileProperty(args.expt_dir, 'workload') if (not workload in trafgen_workloads and not workload in memcached_workloads): print 'Workload not recognized for expt: %s' % args.expt_dir sys.exit(1) # Create directory for pickled files pickle_dir = os.path.join(args.expt_dir, 'pickled') if not os.path.exists(pickle_dir): os.makedirs(pickle_dir) # Create directory for saving statistics stats_dir = os.path.join(args.expt_dir, 'stats') if not os.path.exists(stats_dir): os.makedirs(stats_dir) # Pickle data for trafgen workloads if workload in trafgen_workloads: # Temp directory to extract the sniffer data and pickle it snf_data_dir = os.path.join(args.tmp_dir, 'snf_data') if not os.path.exists(snf_data_dir): os.makedirs(snf_data_dir) # Extract the sniffer data to the temp directory snf_tarfile = os.path.join(args.expt_dir, 'logs/pkt_snf.tar.gz') tar = tarfile.open(snf_tarfile) tar.extractall(snf_data_dir) tar.close() # Pickle sniffer data if required if (args.force_rewrite or not allFilesGenerated('sniffer', pickle_dir, stats_dir)): pickleSnfFile(os.path.join(snf_data_dir, 'pkt_snf.txt'), pickle_dir, stats_dir, max_lines = 1000000) # Pickle mpstat data if (args.force_rewrite or not allFilesGenerated('mpstat', pickle_dir, stats_dir)): pickleMPStat(os.path.join(args.expt_dir, 'logs/mpstat.txt'), pickle_dir, stats_dir) # Pickle ethstats data if (args.force_rewrite or not allFilesGenerated('ethstats', pickle_dir, stats_dir)): pickleEthstats(os.path.join(args.expt_dir, 'logs/net.txt'), pickle_dir, stats_dir) # Pickle data for memcached workloads elif workload in memcached_workloads: # Read the list of server and client machines used for the experiment (servers, clients) = parseHostsFile(os.path.join(args.expt_dir, 'logs/hostsfile.txt')) # Pickle mcperf data if (args.force_rewrite or not allFilesGenerated('mcperf', pickle_dir, stats_dir)): # Pickle hosts info separately hosts_pfile = os.path.join(pickle_dir, 'hosts_p.txt') fd = open(hosts_pfile, 'wb') cPickle.dump((servers, clients), fd) fd.close() mcperf_files = [] for client in clients: files = glob.glob(os.path.join(args.expt_dir, 'logs', client, 'mcperf-t*-c*-*.txt')) mcperf_files.append((client, files)) pickleMcperf(mcperf_files, pickle_dir, stats_dir) # Pickle mpstat data for clients and servers if (args.force_rewrite or not allFilesGenerated('mpstat_mc', pickle_dir, stats_dir)): client_mpstat_files = [ os.path.join(args.expt_dir, 'logs', client, 'mpstat.txt') for client in clients ] server_mpstat_files = [ os.path.join(args.expt_dir, 'logs', server, 'mpstat.txt') for server in servers ] pickleMPStatMC(client_mpstat_files, server_mpstat_files, pickle_dir, stats_dir) # Pickle trafgen data if (args.force_rewrite or not allFilesGenerated('trafgen', pickle_dir, stats_dir)): hosts = clients + servers pickleTrafgen(hosts, os.path.join(args.expt_dir, 'logs'), pickle_dir, stats_dir)
def main(argv): # Parse flags args = parser.parse_args() # Check if the base directory is valid exists = os.path.exists(args.base_dir) isdir = os.path.isdir(args.base_dir) if not exists: print 'Base directory does not exist' sys.exit(1) if not isdir: print 'Base directory not valid' sys.exit(1) args.base_dir = os.path.abspath(args.base_dir) # Create Plumbum commands for pickling data and plotting pickle_cmd = plumbum.local[os.path.join(config['PLOT_SCRIPTS_DIR'], 'pickleExptLogs.py')] plot_sniffer_cmd = plumbum.local[os.path.join(config['PLOT_SCRIPTS_DIR'], 'plotSniffer.py')] plot_mcperf_lat_cmd = plumbum.local[os.path.join(config['PLOT_SCRIPTS_DIR'], 'plotMcperfLatency.py')] # Temp directory for plotting graphs plot_tmpdir = config['PLOT_TMPDIR'] print 'Exploring base directory :', args.base_dir num_exp_dirs = 0 for (path, dirs, files) in os.walk(args.base_dir, followlinks=True): # Check if an experiment directory was found if os.path.exists(os.path.join(path, 'expsift_tags')): print 'Found experiment directory:', path num_exp_dirs += 1 # Pickle experiment logs print '... Pickling experiment data' if args.force_replot: pickle_cmd('-f', path, plot_tmpdir) else: pickle_cmd(path, plot_tmpdir) # Read the workload type for the experiment directory workload = readDirTagFileProperty(path, 'workload') if (not workload in trafgen_workloads and not workload in memcached_workloads): print 'Workload not recognized for expt: %s' % args.expt_dir sys.exit(1) # Plot experiment graphs print '... Plotting experiment graphs' expt_plot_dir = os.path.join(path, 'plot/') if not os.path.exists(expt_plot_dir): os.makedirs(expt_plot_dir) if workload in trafgen_workloads: if args.force_replot: plot_sniffer_cmd('-f', path, expt_plot_dir) else: plot_sniffer_cmd(path, expt_plot_dir) elif workload in memcached_workloads: if args.force_replot: plot_mcperf_lat_cmd('-f', path, expt_plot_dir) else: plot_mcperf_lat_cmd(path, expt_plot_dir) print 'Plotted graphs for %d experiments under %s' % (num_exp_dirs, args.base_dir)
def main(argv): # Parse flags args = parser.parse_args() # Read the workload type for the experiment directory workload = readDirTagFileProperty(args.expt_dir, 'workload') if (not workload in trafgen_workloads and not workload in memcached_workloads): print 'Workload not recognized for expt: %s' % args.expt_dir sys.exit(1) # Create directory for pickled files pickle_dir = os.path.join(args.expt_dir, 'pickled') if not os.path.exists(pickle_dir): os.makedirs(pickle_dir) # Create directory for saving statistics stats_dir = os.path.join(args.expt_dir, 'stats') if not os.path.exists(stats_dir): os.makedirs(stats_dir) # Pickle data for trafgen workloads if workload in trafgen_workloads: # Temp directory to extract the sniffer data and pickle it snf_data_dir = os.path.join(args.tmp_dir, 'snf_data') if not os.path.exists(snf_data_dir): os.makedirs(snf_data_dir) # Extract the sniffer data to the temp directory snf_tarfile = os.path.join(args.expt_dir, 'logs/pkt_snf.tar.gz') tar = tarfile.open(snf_tarfile) tar.extractall(snf_data_dir) tar.close() # Pickle sniffer data if required if (args.force_rewrite or not allFilesGenerated('sniffer', pickle_dir, stats_dir)): pickleSnfFile(os.path.join(snf_data_dir, 'pkt_snf.txt'), pickle_dir, stats_dir, max_lines=1000000) # Pickle mpstat data if (args.force_rewrite or not allFilesGenerated('mpstat', pickle_dir, stats_dir)): pickleMPStat(os.path.join(args.expt_dir, 'logs/mpstat.txt'), pickle_dir, stats_dir) # Pickle ethstats data if (args.force_rewrite or not allFilesGenerated('ethstats', pickle_dir, stats_dir)): pickleEthstats(os.path.join(args.expt_dir, 'logs/net.txt'), pickle_dir, stats_dir) # Pickle data for memcached workloads elif workload in memcached_workloads: # Read the list of server and client machines used for the experiment (servers, clients) = parseHostsFile( os.path.join(args.expt_dir, 'logs/hostsfile.txt')) # Pickle mcperf data if (args.force_rewrite or not allFilesGenerated('mcperf', pickle_dir, stats_dir)): # Pickle hosts info separately hosts_pfile = os.path.join(pickle_dir, 'hosts_p.txt') fd = open(hosts_pfile, 'wb') cPickle.dump((servers, clients), fd) fd.close() mcperf_files = [] for client in clients: files = glob.glob( os.path.join(args.expt_dir, 'logs', client, 'mcperf-t*-c*-*.txt')) mcperf_files.append((client, files)) pickleMcperf(mcperf_files, pickle_dir, stats_dir) # Pickle mpstat data for clients and servers if (args.force_rewrite or not allFilesGenerated('mpstat_mc', pickle_dir, stats_dir)): client_mpstat_files = [ os.path.join(args.expt_dir, 'logs', client, 'mpstat.txt') for client in clients ] server_mpstat_files = [ os.path.join(args.expt_dir, 'logs', server, 'mpstat.txt') for server in servers ] pickleMPStatMC(client_mpstat_files, server_mpstat_files, pickle_dir, stats_dir) # Pickle trafgen data if (args.force_rewrite or not allFilesGenerated('trafgen', pickle_dir, stats_dir)): hosts = clients + servers pickleTrafgen(hosts, os.path.join(args.expt_dir, 'logs'), pickle_dir, stats_dir)