def testParsesHistogram(self):
expected = {
300: 5771,
400: 118948,
500: 7121,
600: 639,
700: 199,
800: 90,
900: 53,
1000: 149,
2000: 31,
3000: 11,
4000: 8,
5000: 1,
6000: 1,
7000: 1,
9000: 1
}
hist = netperf.ParseHistogram(self.netperf_output)
self.assertEqual(hist, expected)
def ParseNetperfOutput(stdout, metadata, benchmark_name,
enable_latency_histograms):
"""Parses the stdout of a single netperf process.
Args:
stdout: the stdout of the netperf process
metadata: metadata for any sample.Sample objects we create
Returns:
A tuple containing (throughput_sample, latency_samples, latency_histogram)
"""
# Don't modify the metadata dict that was passed in
metadata = metadata.copy()
# Extract stats from stdout
# Sample output:
#
# "MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 20001
# AF_INET to 104.154.50.86 () port 20001 AF_INET : +/-2.500% @ 99% conf.
# : first burst 0",\n
# Throughput,Throughput Units,Throughput Confidence Width (%),
# Confidence Iterations Run,Stddev Latency Microseconds,
# 50th Percentile Latency Microseconds,90th Percentile Latency Microseconds,
# 99th Percentile Latency Microseconds,Minimum Latency Microseconds,
# Maximum Latency Microseconds\n
# 1405.50,Trans/s,2.522,4,783.80,683,735,841,600,900\n
try:
fp = io.StringIO(stdout)
# "-o" flag above specifies CSV output, but there is one extra header line:
banner = next(fp)
assert banner.startswith('MIGRATED'), stdout
r = csv.DictReader(fp)
results = next(r)
logging.info('Netperf Results: %s', results)
assert 'Throughput' in results
except:
raise Exception('Netperf ERROR: Failed to parse stdout. STDOUT: %s' %
stdout)
# Update the metadata with some additional infos
meta_keys = [('Confidence Iterations Run', 'confidence_iter'),
('Throughput Confidence Width (%)', 'confidence_width_percent')]
metadata.update({meta_key: results[netperf_key]
for netperf_key, meta_key in meta_keys})
# Create the throughput sample
throughput = float(results['Throughput'])
throughput_units = results['Throughput Units']
if throughput_units == '10^6bits/s':
# TCP_STREAM benchmark
unit = MBPS
metric = '%s_Throughput' % benchmark_name
elif throughput_units == 'Trans/s':
# *RR benchmarks
unit = TRANSACTIONS_PER_SECOND
metric = '%s_Transaction_Rate' % benchmark_name
else:
raise ValueError('Netperf output specifies unrecognized throughput units %s'
% throughput_units)
throughput_sample = sample.Sample(metric, throughput, unit, metadata)
latency_hist = None
latency_samples = []
if enable_latency_histograms:
# Parse the latency histogram. {latency: count} where "latency" is the
# latency in microseconds with only 2 significant figures and "count" is the
# number of response times that fell in that latency range.
latency_hist = netperf.ParseHistogram(stdout)
hist_metadata = {'histogram': json.dumps(latency_hist)}
hist_metadata.update(metadata)
latency_samples.append(sample.Sample(
'%s_Latency_Histogram' % benchmark_name, 0, 'us', hist_metadata))
if unit != MBPS:
for metric_key, metric_name in [
('50th Percentile Latency Microseconds', 'p50'),
('90th Percentile Latency Microseconds', 'p90'),
('99th Percentile Latency Microseconds', 'p99'),
('Minimum Latency Microseconds', 'min'),
('Maximum Latency Microseconds', 'max'),
('Stddev Latency Microseconds', 'stddev')]:
if metric_key in results:
latency_samples.append(
sample.Sample('%s_Latency_%s' % (benchmark_name, metric_name),
float(results[metric_key]), 'us', metadata))
return (throughput_sample, latency_samples, latency_hist)
def ParseNetperfOutput(stdout,
metadata,
benchmark_name,
enable_latency_histograms,
instance=None):
"""Parses the stdout of a single netperf process.
Args:
stdout: the stdout of the netperf process
metadata: metadata for any sample.Sample objects we create
benchmark_name: the name of the netperf benchmark
enable_latency_histograms: bool indicating if latency histograms are
included in stdout
instance: on which machines the test has been executed
Returns:
A tuple containing (throughput_sample, latency_samples, latency_histogram)
"""
# Don't modify the metadata dict that was passed in
metadata = metadata.copy()
# Extract stats from stdout
# Sample output:
#
# "MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 20001
# AF_INET to 104.154.50.86 () port 20001 AF_INET : +/-2.500% @ 99% conf.
# : first burst 0",\n
# Throughput,Throughput Units,Throughput Confidence Width (%),
# Confidence Iterations Run,Stddev Latency Microseconds,
# 50th Percentile Latency Microseconds,90th Percentile Latency Microseconds,
# 99th Percentile Latency Microseconds,Minimum Latency Microseconds,
# Maximum Latency Microseconds\n
# 1405.50,Trans/s,2.522,4,783.80,683,735,841,600,900\n
try:
fp = six.StringIO(stdout)
# "-o" flag above specifies CSV output, but there is one extra header line:
banner = next(fp)
assert banner.startswith('MIGRATED'), stdout
r = csv.DictReader(fp)
results = next(r)
logging.info('Netperf Results: %s', results)
assert 'Throughput' in results
except (StopIteration, AssertionError):
# The output returned by netperf was unparseable - usually due to a broken
# connection or other error. Raise KnownIntermittentError to signal the
# benchmark can be retried. Do not automatically retry as an immediate
# retry on these VMs may be adveresly affected (e.g. burstable credits
# partially used)
message = 'Netperf ERROR: Failed to parse stdout. STDOUT: %s' % stdout
logging.error(message)
raise errors.Benchmarks.KnownIntermittentError(message)
# Update the metadata with some additional infos
meta_keys = [('Confidence Iterations Run', 'confidence_iter'),
('Throughput Confidence Width (%)',
'confidence_width_percent')]
if 'TCP' in benchmark_name:
meta_keys.extend([
('Local Transport Retransmissions', 'netperf_retransmissions'),
('Remote Transport Retransmissions', 'netserver_retransmissions'),
])
metadata.update({
meta_key: results[netperf_key]
for netperf_key, meta_key in meta_keys
})
# Create the throughput sample
throughput = float(results['Throughput'])
throughput_units = results['Throughput Units']
if throughput_units == '10^6bits/s':
# TCP_STREAM benchmark
unit = MBPS
metric = '%s_Throughput' % benchmark_name
elif throughput_units == 'Trans/s':
# *RR benchmarks
unit = TRANSACTIONS_PER_SECOND
metric = '%s_Transaction_Rate' % benchmark_name
else:
raise ValueError(
'Netperf output specifies unrecognized throughput units %s' %
throughput_units)
throughput_sample = sample.Sample(metric,
throughput,
unit,
metadata,
instance=instance)
latency_hist = None
latency_samples = []
if enable_latency_histograms:
# Parse the latency histogram. {latency: count} where "latency" is the
# latency in microseconds with only 2 significant figures and "count" is the
# number of response times that fell in that latency range.
latency_hist = netperf.ParseHistogram(stdout)
hist_metadata = {'histogram': json.dumps(latency_hist)}
hist_metadata.update(metadata)
latency_samples.append(
sample.Sample('%s_Latency_Histogram' % benchmark_name,
0,
'us',
hist_metadata,
instance=instance))
if unit != MBPS:
for metric_key, metric_name in [
('50th Percentile Latency Microseconds', 'p50'),
('90th Percentile Latency Microseconds', 'p90'),
('99th Percentile Latency Microseconds', 'p99'),
('Minimum Latency Microseconds', 'min'),
('Maximum Latency Microseconds', 'max'),
('Stddev Latency Microseconds', 'stddev')
]:
if metric_key in results:
latency_samples.append(
sample.Sample('%s_Latency_%s' %
(benchmark_name, metric_name),
float(results[metric_key]),
'us',
metadata,
instance=instance))
return (throughput_sample, latency_samples, latency_hist)
def RunNetperf(vm, benchmark_name, server_ip):
"""Spawns netperf on a remove VM, parses results.
Args:
vm: The VM that the netperf TCP_RR benchmark will be run upon.
benchmark_name: The netperf benchmark to run, see the documentation.
server_ip: A machine that is running netserver.
Returns:
A sample.Sample object with the result.
"""
# Flags:
# -o specifies keys to include in CSV output.
# -j keeps additional latency numbers
# -v sets the verbosity level so that netperf will print out histograms
# -I specifies the confidence % and width - here 99% confidence that the true
# value is within +/- 2.5% of the reported value
# -i specifies the maximum and minimum number of iterations.
confidence = ('-I 99,5 -i {0},3'.format(FLAGS.netperf_max_iter)
if FLAGS.netperf_max_iter else '')
verbosity = '-v2 ' if FLAGS.netperf_enable_histograms else ''
netperf_cmd = (
'{netperf_path} -p {command_port} -j {verbosity}'
'-t {benchmark_name} -H {server_ip} -l {length} {confidence} '
' -- '
'-P {data_port} '
'-o THROUGHPUT,THROUGHPUT_UNITS,P50_LATENCY,P90_LATENCY,'
'P99_LATENCY,STDDEV_LATENCY,'
'MIN_LATENCY,MAX_LATENCY,'
'CONFIDENCE_ITERATION,THROUGHPUT_CONFID').format(
netperf_path=netperf.NETPERF_PATH,
benchmark_name=benchmark_name,
server_ip=server_ip,
command_port=COMMAND_PORT,
data_port=DATA_PORT,
length=FLAGS.netperf_test_length,
confidence=confidence,
verbosity=verbosity)
stdout, _ = vm.RemoteCommand(netperf_cmd,
timeout=2 * FLAGS.netperf_test_length *
(FLAGS.netperf_max_iter or 1))
fp = io.StringIO(stdout)
# "-o" flag above specifies CSV output, but there is one extra header line:
banner = next(fp)
assert banner.startswith('MIGRATED'), stdout
r = csv.DictReader(fp)
row = next(r)
logging.info('Netperf Results: %s', row)
assert 'Throughput' in row, row
value = float(row['Throughput'])
unit = {
'Trans/s': TRANSACTIONS_PER_SECOND,
'10^6bits/s': MBPS
}[row['Throughput Units']]
if unit == MBPS:
metric = '%s_Throughput' % benchmark_name
else:
metric = '%s_Transaction_Rate' % benchmark_name
meta_keys = [('Confidence Iterations Run', 'confidence_iter'),
('Throughput Confidence Width (%)',
'confidence_width_percent')]
metadata = {meta_key: row[np_key] for np_key, meta_key in meta_keys}
metadata.update(netperf_test_length=FLAGS.netperf_test_length,
max_iter=FLAGS.netperf_max_iter or 1)
samples = [sample.Sample(metric, value, unit, metadata)]
# No tail latency for throughput.
if unit == MBPS:
return samples
if FLAGS.netperf_enable_histograms:
# Generate a sample containing the entire histogram of
# latencies.
hist = netperf.ParseHistogram(stdout)
hist_metadata = {'histogram': json.dumps(hist)}
hist_metadata.update(metadata)
samples.append(
sample.Sample('%s_Latency_Histogram' % benchmark_name, 0, 'us',
hist_metadata))
for metric_key, metric_name in [
('50th Percentile Latency Microseconds', 'p50'),
('90th Percentile Latency Microseconds', 'p90'),
('99th Percentile Latency Microseconds', 'p99'),
('Minimum Latency Microseconds', 'min'),
('Maximum Latency Microseconds', 'max'),
('Stddev Latency Microseconds', 'stddev')
]:
samples.append(
sample.Sample('%s_Latency_%s' % (benchmark_name, metric_name),
float(row[metric_key]), 'us', metadata))
return samples
