def MakeAccuracySamplesFromOutput(base_metadata: Dict[str, Any],
output: str) -> List[sample.Sample]:
"""Create accuracy samples containing metrics.
Args:
base_metadata: dict contains all the metadata that reports.
output: string, command output
Returns:
Samples containing training metrics.
"""
metadata = {}
for column_name in _ACCURACY_METADATA:
metadata[f'mlperf {column_name}'] = regex_util.ExtractExactlyOneMatch(
fr'{re.escape(column_name)} *: *(.*)', output)
accuracy = regex_util.ExtractFloat(
r': Accuracy = (\d+\.\d+), Threshold = \d+\.\d+\. Accuracy test PASSED',
output)
metadata['Threshold'] = regex_util.ExtractFloat(
r': Accuracy = \d+\.\d+, Threshold = (\d+\.\d+)\. Accuracy test PASSED',
output)
metadata.update(base_metadata)
return [sample.Sample('accuracy', float(accuracy), '%', metadata)]
def _ParseInterruptsPerSec(rows: List[str],
metadata: Dict[str, Any],
timestamp: Optional[float] = None):
"""Parse a interrput/sec data chunk.
Args:
rows: List of mpstat interrupts per second lines.
metadata: metadata of the sample.
timestamp: timestamp of the sample.
Yields:
List of samples
input data:
Average: CPU intr/s
Average: all 3371.98
Average: 0 268.54
Average: 1 265.59
...
"""
for row in rows[1:]: # skipping first header row
data = row.split()
meta = metadata.copy()
if 'all' in data:
metric_name = 'mpstat_avg_intr'
cpu_id = -1
else:
metric_name = 'mpstat_intr'
cpu_id = int(data[1])
meta['mpstat_cpu_id'] = cpu_id
yield sample.Sample(
metric=metric_name,
value=float(data[2]),
unit='interrupts/sec',
metadata=meta,
timestamp=timestamp,
)
def _RunIperf(sending_vm, receiving_vm, receiving_ip_address, ip_type):
"""Run iperf using sending 'vm' to connect to 'ip_address'.
Args:
sending_vm: The VM sending traffic.
receiving_vm: The VM receiving traffic.
receiving_ip_address: The IP address of the iperf server (ie the receiver).
ip_type: The IP type of 'ip_address' (e.g. 'internal', 'external')
Returns:
A Sample.
Raises:
regex_util.NoMatchError: When iperf results are not found in stdout.
"""
iperf_cmd = ('iperf --client %s --port %s --format m --time 60' %
(receiving_ip_address, IPERF_PORT))
stdout, _ = sending_vm.RemoteCommand(iperf_cmd, should_log=True)
iperf_pattern = re.compile(r'(\d+\.\d+|\d+) Mbits/sec')
value = regex_util.ExtractFloat(iperf_pattern, stdout)
metadata = {
# TODO(voellm): The server and client terminology is being
# deprecated. It does not make clear the direction of the flow.
'server_machine_type': receiving_vm.machine_type,
'server_zone': receiving_vm.zone,
'client_machine_type': sending_vm.machine_type,
'client_zone': sending_vm.zone,
# The meta data defining the environment
'receiving_machine_type': receiving_vm.machine_type,
'receiving_zone': receiving_vm.zone,
'sending_machine_type': sending_vm.machine_type,
'sending_zone': sending_vm.zone,
'ip_type': ip_type
}
return sample.Sample('Throughput', float(value), 'Mbits/sec', metadata)
def ParseOutput(hpcc_output, benchmark_spec):
"""Parses the output from HPCC.
Args:
hpcc_output: A string containing the text of hpccoutf.txt.
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
Returns:
A list of samples to be published (in the same format as Run() returns).
"""
results = []
base_metadata = {
'num_machines': len(benchmark_spec.vms)
}
UpdateMetadata(base_metadata)
base_metadata.update(benchmark_spec.hpcc_dimensions)
# Parse all metrics from metric=value lines in the HPCC output.
metric_values = regex_util.ExtractAllFloatMetrics(hpcc_output)
# For each benchmark that is run, collect the metrics and metadata for that
# benchmark from the metric_values map.
benchmarks_run = FLAGS.hpcc_benchmarks or hpcc.HPCC_METRIC_MAP
for benchmark in benchmarks_run:
for metric, units in six.iteritems(hpcc.HPCC_METRIC_MAP[benchmark]):
value = metric_values[metric]
# Copy metadata reported in the HPCC summary statistics to the metadata.
metadata = base_metadata.copy()
for metadata_item in hpcc.HPCC_METADATA_MAP[benchmark]:
metadata[metadata_item] = metric_values[metadata_item]
results.append(sample.Sample(metric, value, units, metadata))
return results
def _ParsePercentageUse(rows, metadata):
"""Parse a CPU percentage use data chunk.
Args:
rows: List of mpstat CPU percentage lines.
metadata: metadata of the sample.
Yields:
List of samples
input data:
Average: CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle
Average: all 1.82 0.11 0.84 0.05 0.00 0.31 0.00 0.00 0.00 96.88
Average: 0 1.77 0.09 0.82 0.07 0.00 2.21 0.00 0.00 0.00 95.04
Average: 1 1.85 0.12 0.83 0.06 0.00 0.65 0.00 0.00 0.00 96.49
...
"""
header_row = rows[0]
headers = [header.strip('%') for header in header_row.split()]
for row in rows[1:]:
data = row.split()
name_value_pairs = list(zip(headers, data))
cpu_id_pair = name_value_pairs[1]
for header, value in name_value_pairs[2:]:
meta = metadata.copy()
if 'all' in cpu_id_pair:
metric_name = 'mpstat_avg_' + header
cpu_id = -1
else:
metric_name = 'mpstat_' + header
cpu_id = int(cpu_id_pair[1])
meta['mpstat_cpu_id'] = cpu_id
yield sample.Sample(metric=metric_name,
value=float(value),
unit='%',
metadata=meta)
def Run(benchmark_spec):
"""Run Coremark on the target vm.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
Returns:
A list of sample.Sample objects.
"""
vms = benchmark_spec.vms
vm = vms[0]
logging.info('Coremark running on %s', vm)
num_cpus = vm.num_cpus
vm.RemoteCommand(
'cd %s;make PORT_DIR=linux64 ITERATIONS=%s XCFLAGS="-g -O2 '
'-DMULTITHREAD=%d -DUSE_PTHREAD -DPERFORMANCE_RUN=1"' %
(COREMARK_DIR, ITERATIONS_PER_CPU, num_cpus))
logging.info('Coremark Results:')
stdout, _ = vm.RemoteCommand('cat %s/run1.log' % COREMARK_DIR,
should_log=True)
value = regex_util.ExtractFloat(r'CoreMark 1.0 : ([0-9]*\.[0-9]*)', stdout)
metadata = {'num_cpus': vm.num_cpus}
return [sample.Sample('Coremark Score', value, '', metadata)]
def Run(benchmark_spec):
"""Run the data analytics benchmark.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
Returns:
A list of sample.Sample objects.
"""
master = benchmark_spec.vm_groups['master'][0]
results = []
stdout, _ = master.RemoteCommand('sudo docker exec master benchmark 2>&1',
should_log=True)
matches = re.findall(r'^Benchmark time: (\d+)ms$', stdout, re.MULTILINE)
if len(matches) != 1:
raise errors.Benchmark.RunError('Expected to find benchmark runtime')
results.append(
sample.Sample('Benchmark runtime',
float(matches[0]) / 1000, 'seconds'))
return results
def _CreateSingleSample(sample_name, sample_units, metadata, client_stdout):
"""Creates a sample from the tensorflow_serving_client_workload stdout.
client_stdout is expected to contain output in the following format:
key1: int_or_float_value_1
key2: int_or_float_value_2
Args:
sample_name: Name of the sample. Used to create a regex to extract the value
from client_stdout. Also used as the returned sample's name.
sample_units: Units to be specified in the returned sample
metadata: Metadata to be added to the returned sample
client_stdout: Stdout from tensorflow_serving_client_workload.py
Returns:
A single floating point sample.
Raises:
regex_util.NoMatchError: when no line beginning with sample_name: is found
in client_stdout
"""
regex = sample_name + r'\:\s*(\w+\.?\w*)'
value = regex_util.ExtractFloat(regex, client_stdout)
return sample.Sample(sample_name, value, sample_units, metadata)
def RunCp(vms, data_size_in_mb, metadata):
"""Runs cp benchmarks and parses results.
Args:
vms: The VMs running cp benchmarks.
data_size_in_mb: The size of the data file in MB.
metadata: The base metadata to attach to the sample.
Returns:
A list of sample.Sample objects.
"""
cmd = ('rm -rf %s/*; sudo sync; sudo sysctl vm.drop_caches=3; '
'time cp %s/data/* %s/; ' %
(vms[0].GetScratchDir(1), vms[0].GetScratchDir(0),
vms[0].GetScratchDir(1)))
_, res = vms[0].RemoteCommand(cmd)
logging.info(res)
time_used = vm_util.ParseTimeCommandResult(res)
return [
sample.Sample('cp throughput',
data_size_in_mb / time_used,
UNIT,
metadata=metadata)
]
def get_aggregated_query_performance_sample(
self, query_name: Text, metadata: Dict[str, str]) -> sample.Sample:
"""Get the performance of query aggregated across all iterations.
Args:
query_name: Name of the query whose aggregated performance is requested
metadata: A dictionary of execution attributes to be merged with the query
execution attributes, for eg. tpc suite, scale of dataset, etc.
Returns:
A sample of the query's aggregated execution time
"""
query_metadata = copy.copy(metadata)
query_metadata['query'] = query_name
query_metadata['aggregation_method'] = 'mean'
perf, exec_status, agg_md = -1.0, EdwQueryExecutionStatus.FAILED, {}
if self.aggregated_query_status(query_name):
perf = self.aggregated_query_execution_time(query_name=query_name)
exec_status = EdwQueryExecutionStatus.SUCCESSFUL
agg_md = self.aggregated_query_metadata(query_name=query_name)
query_metadata['execution_status'] = exec_status
query_metadata.update(agg_md)
return sample.Sample('edw_aggregated_query_time', perf, 'seconds',
query_metadata)
def get_queries_geomean_performance_sample(
self, expected_queries: List[Text], metadata: Dict[str,
str]) -> sample.Sample:
"""Gets a sample for geomean of all queries in all streams of the iteration.
Args:
expected_queries: A list of query names expected to have been executed in
an iteration.
metadata: A dictionary of execution attributes to be merged with the query
execution attributes, for eg. tpc suite, scale of dataset, etc.
Returns:
A sample of iteration geomean performance.
Raises:
EdwPerformanceAggregationError: If the iteration contains unsuccessful
query executions.
"""
if not self.is_successful(expected_queries):
raise EdwPerformanceAggregationError('Failed executions in iteration.')
raw_geo_mean = self.get_queries_geomean()
geo_mean_metadata = copy.copy(metadata)
return sample.Sample('edw_iteration_geomean_time', raw_geo_mean, 'seconds',
geo_mean_metadata)
def _ParseOutput(output_str):
numbers = [
float(f) for f in re.findall(r"([-+]?\d*\.\d+|\d+)", " ".join(
output_str.splitlines(1)[-4:]))
]
results = []
results.append(sample.Sample("Requests per second", numbers[1], "req/s"))
results.append(sample.Sample("Average latency", numbers[7], "ms"))
results.append(sample.Sample("90th percentile latency", numbers[8], "ms"))
results.append(sample.Sample("95th percentile latency", numbers[9], "ms"))
results.append(sample.Sample("99th percentile latency", numbers[10], "ms"))
req_rems = numbers[15:-1]
results.append(
sample.Sample("Average outstanding requests per requester",
sum(req_rems) / len(req_rems), "reqs"))
return results
def GetSamples(self, metadata: Dict[str, Any]) -> List[sample.Sample]:
"""Return this result as a list of samples."""
metadata['avg_latency'] = self.latency_ms
metadata['p90_latency'] = self.p90_latency
metadata['p95_latency'] = self.p95_latency
metadata['p99_latency'] = self.p99_latency
samples = [
sample.Sample('Ops Throughput', self.ops_per_sec, 'ops/s',
metadata),
sample.Sample('KB Throughput', self.kb_per_sec, 'KB/s', metadata),
sample.Sample('Latency', self.latency_ms, 'ms', metadata),
]
for name, histogram in [('get', self.get_latency_histogram),
('set', self.set_latency_histogram)]:
hist_meta = copy.deepcopy(metadata)
hist_meta.update({'histogram': json.dumps(histogram)})
samples.append(
sample.Sample(f'{name} latency histogram', 0, '', hist_meta))
if self.ops_time_series:
ops_time_series_dict = {}
for interval, count in self.ops_time_series:
ops_time_series_dict[interval] = count
ops_series_metadata = copy.deepcopy(metadata)
ops_series_metadata.update({'time_series': ops_time_series_dict})
samples.append(
sample.Sample('Ops Time Series', 0, 'ops',
ops_series_metadata))
if self.max_latency_time_series:
latency_time_series_dict = {}
for interval, latency in self.max_latency_time_series:
latency_time_series_dict[interval] = latency
latency_series_metadata = copy.deepcopy(metadata)
latency_series_metadata.update(
{'time_series': latency_time_series_dict})
samples.append(
sample.Sample('Max Latency Time Series', 0, 'ms',
latency_series_metadata))
return samples
def testGenerateSamplesRuntimeAndTimestamps(self):
"""Test generating both runtime and timestamp samples."""
timer = timing_util.IntervalTimer()
with timer.Measure('First'):
pass
with timer.Measure('Second'):
pass
start0 = timer.intervals[0][1]
stop0 = timer.intervals[0][2]
start1 = timer.intervals[1][1]
stop1 = timer.intervals[1][2]
samples = timer.GenerateSamples(include_runtime=True,
include_timestamps=True)
exp_samples = [
sample.Sample('First Runtime', stop0 - start0, 'seconds'),
sample.Sample('First Start Timestamp', start0, 'seconds'),
sample.Sample('First Stop Timestamp', stop0, 'seconds'),
sample.Sample('Second Runtime', stop1 - start1, 'seconds'),
sample.Sample('Second Start Timestamp', start1, 'seconds'),
sample.Sample('Second Stop Timestamp', stop1, 'seconds')
]
self.assertSampleListsEqualUpToTimestamp(samples, exp_samples)
def testGenerateSamplesRuntimeAndTimestamps(self):
"""Test generating both runtime and timestamp samples."""
timer = timing_util.IntervalTimer()
with timer.Measure('First'):
pass
with timer.Measure('Second'):
pass
start0 = timer.intervals[0][1]
stop0 = timer.intervals[0][2]
start1 = timer.intervals[1][1]
stop1 = timer.intervals[1][2]
with mock.patch(
'perfkitbenchmarker.timing_util.TimestampMeasurementsEnabled',
return_value=True):
samples = timer.GenerateSamples()
exp_samples = [
sample.Sample('First Runtime', stop0 - start0, 'seconds'),
sample.Sample('First Start Timestamp', start0, 'seconds'),
sample.Sample('First Stop Timestamp', stop0, 'seconds'),
sample.Sample('Second Runtime', stop1 - start1, 'seconds'),
sample.Sample('Second Start Timestamp', start1, 'seconds'),
sample.Sample('Second Stop Timestamp', stop1, 'seconds')]
self.assertSampleListsEqualUpToTimestamp(samples, exp_samples)
def testIperfTCPMultiStream(self):
tcp_number_of_streams = 10
fake_vm = namedtuple('fake_vm', 'machine_type zone')
sending_vm = fake_vm(machine_type='A', zone='B')
receiving_vm = fake_vm(machine_type='A', zone='B')
def _Metadata(thread_id):
return {
'protocol': 'TCP',
'num_threads': tcp_number_of_streams,
'receiving_machine_type': receiving_vm.machine_type,
'receiving_zone': receiving_vm.zone,
'sending_machine_type': sending_vm.machine_type,
'sending_zone': sending_vm.zone,
'thread_id': thread_id,
'internal_ip_used': True,
'tcp_window_size': None,
}
expected_samples = [
sample.Sample('Bandwidth', 935.0, 'Mbits/sec', _Metadata('4')),
sample.Sample('Bandwidth', 935.0, 'Mbits/sec', _Metadata('6')),
sample.Sample('Bandwidth', 935.0, 'Mbits/sec', _Metadata('8')),
sample.Sample('Bandwidth', 935.0, 'Mbits/sec', _Metadata('10')),
sample.Sample('Bandwidth', 935.0, 'Mbits/sec', _Metadata('12')),
sample.Sample('Bandwidth', 4670.0, 'Mbits/sec', _Metadata('SUM')),
]
samples = iperf3.ParseTCPMultiStreamOutput(iperf3_tcp_results,
sending_vm, receiving_vm,
tcp_number_of_streams, True)
self.assertSampleListsEqualUpToTimestamp(samples, expected_samples)
def testMetadataOptional(self):
instance = sample.Sample(metric='Test', value=1.0, unit='Mbps')
self.assertDictEqual({}, instance.metadata)
def RunNetperf(vm, benchmark_name, server_ip, num_streams):
"""Spawns netperf on a remote VM, parses results.
Args:
vm: The VM that the netperf TCP_RR benchmark will be run upon.
benchmark_name: The netperf benchmark to run, see the documentation.
server_ip: A machine that is running netserver.
num_streams: The number of netperf client threads to run.
Returns:
A sample.Sample object with the result.
"""
enable_latency_histograms = FLAGS.netperf_enable_histograms or num_streams > 1
# Throughput benchmarks don't have latency histograms
enable_latency_histograms = enable_latency_histograms and \
benchmark_name != 'TCP_STREAM'
# Flags:
# -o specifies keys to include in CSV output.
# -j keeps additional latency numbers
# -v sets the verbosity level so that netperf will print out histograms
# -I specifies the confidence % and width - here 99% confidence that the true
# value is within +/- 2.5% of the reported value
# -i specifies the maximum and minimum number of iterations.
confidence = ('-I 99,5 -i {0},3'.format(FLAGS.netperf_max_iter)
if FLAGS.netperf_max_iter else '')
verbosity = '-v2 ' if enable_latency_histograms else ''
netperf_cmd = ('{netperf_path} -p {{command_port}} -j {verbosity} '
'-t {benchmark_name} -H {server_ip} -l {length} {confidence}'
' -- '
'-P ,{{data_port}} '
'-o THROUGHPUT,THROUGHPUT_UNITS,P50_LATENCY,P90_LATENCY,'
'P99_LATENCY,STDDEV_LATENCY,'
'MIN_LATENCY,MAX_LATENCY,'
'CONFIDENCE_ITERATION,THROUGHPUT_CONFID').format(
netperf_path=netperf.NETPERF_PATH,
benchmark_name=benchmark_name,
server_ip=server_ip,
length=FLAGS.netperf_test_length,
confidence=confidence, verbosity=verbosity)
if FLAGS.netperf_thinktime != 0:
netperf_cmd += (' -X {thinktime},{thinktime_array_size},'
'{thinktime_run_length} ').format(
thinktime=FLAGS.netperf_thinktime,
thinktime_array_size=FLAGS.netperf_thinktime_array_size,
thinktime_run_length=FLAGS.netperf_thinktime_run_length)
# Run all of the netperf processes and collect their stdout
# TODO: Record process start delta of netperf processes on the remote machine
# Give the remote script the max possible test length plus 5 minutes to
# complete
remote_cmd_timeout = \
FLAGS.netperf_test_length * (FLAGS.netperf_max_iter or 1) + 300
remote_cmd = ('./%s --netperf_cmd="%s" --num_streams=%s --port_start=%s' %
(REMOTE_SCRIPT, netperf_cmd, num_streams, PORT_START))
remote_stdout, _ = vm.RemoteCommand(remote_cmd,
timeout=remote_cmd_timeout)
# Decode stdouts, stderrs, and return codes from remote command's stdout
json_out = json.loads(remote_stdout)
stdouts = json_out[0]
# Metadata to attach to samples
metadata = {'netperf_test_length': FLAGS.netperf_test_length,
'max_iter': FLAGS.netperf_max_iter or 1,
'sending_thread_count': num_streams}
parsed_output = [ParseNetperfOutput(stdout, metadata, benchmark_name,
enable_latency_histograms)
for stdout in stdouts]
if len(parsed_output) == 1:
# Only 1 netperf thread
throughput_sample, latency_samples, histogram = parsed_output[0]
return [throughput_sample] + latency_samples
else:
# Multiple netperf threads
samples = []
# Unzip parsed output
# Note that latency_samples are invalid with multiple threads because stats
# are computed per-thread by netperf, so we don't use them here.
throughput_samples, _, latency_histograms = [list(t)
for t in zip(*parsed_output)]
# They should all have the same units
throughput_unit = throughput_samples[0].unit
# Extract the throughput values from the samples
throughputs = [s.value for s in throughput_samples]
# Compute some stats on the throughput values
throughput_stats = sample.PercentileCalculator(throughputs, [50, 90, 99])
throughput_stats['min'] = min(throughputs)
throughput_stats['max'] = max(throughputs)
# Calculate aggregate throughput
throughput_stats['total'] = throughput_stats['average'] * len(throughputs)
# Create samples for throughput stats
for stat, value in throughput_stats.items():
samples.append(
sample.Sample('%s_Throughput_%s' % (benchmark_name, stat),
float(value),
throughput_unit, metadata))
if enable_latency_histograms:
# Combine all of the latency histogram dictionaries
latency_histogram = Counter()
for histogram in latency_histograms:
latency_histogram.update(histogram)
# Create a sample for the aggregate latency histogram
hist_metadata = {'histogram': json.dumps(latency_histogram)}
hist_metadata.update(metadata)
samples.append(sample.Sample(
'%s_Latency_Histogram' % benchmark_name, 0, 'us', hist_metadata))
# Calculate stats on aggregate latency histogram
latency_stats = _HistogramStatsCalculator(latency_histogram, [50, 90, 99])
# Create samples for the latency stats
for stat, value in latency_stats.items():
samples.append(
sample.Sample('%s_Latency_%s' % (benchmark_name, stat),
float(value),
'us', metadata))
return samples
def ParseNetperfOutput(stdout, metadata, benchmark_name,
enable_latency_histograms):
"""Parses the stdout of a single netperf process.
Args:
stdout: the stdout of the netperf process
metadata: metadata for any sample.Sample objects we create
Returns:
A tuple containing (throughput_sample, latency_samples, latency_histogram)
"""
# Don't modify the metadata dict that was passed in
metadata = metadata.copy()
# Extract stats from stdout
# Sample output:
#
# "MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 20001
# AF_INET to 104.154.50.86 () port 20001 AF_INET : +/-2.500% @ 99% conf.
# : first burst 0",\n
# Throughput,Throughput Units,Throughput Confidence Width (%),
# Confidence Iterations Run,Stddev Latency Microseconds,
# 50th Percentile Latency Microseconds,90th Percentile Latency Microseconds,
# 99th Percentile Latency Microseconds,Minimum Latency Microseconds,
# Maximum Latency Microseconds\n
# 1405.50,Trans/s,2.522,4,783.80,683,735,841,600,900\n
try:
fp = io.StringIO(stdout)
# "-o" flag above specifies CSV output, but there is one extra header line:
banner = next(fp)
assert banner.startswith('MIGRATED'), stdout
r = csv.DictReader(fp)
results = next(r)
logging.info('Netperf Results: %s', results)
assert 'Throughput' in results
except:
raise Exception('Netperf ERROR: Failed to parse stdout. STDOUT: %s' %
stdout)
# Update the metadata with some additional infos
meta_keys = [('Confidence Iterations Run', 'confidence_iter'),
('Throughput Confidence Width (%)', 'confidence_width_percent')]
metadata.update({meta_key: results[netperf_key]
for netperf_key, meta_key in meta_keys})
# Create the throughput sample
throughput = float(results['Throughput'])
throughput_units = results['Throughput Units']
if throughput_units == '10^6bits/s':
# TCP_STREAM benchmark
unit = MBPS
metric = '%s_Throughput' % benchmark_name
elif throughput_units == 'Trans/s':
# *RR benchmarks
unit = TRANSACTIONS_PER_SECOND
metric = '%s_Transaction_Rate' % benchmark_name
else:
raise ValueError('Netperf output specifies unrecognized throughput units %s'
% throughput_units)
throughput_sample = sample.Sample(metric, throughput, unit, metadata)
latency_hist = None
latency_samples = []
if enable_latency_histograms:
# Parse the latency histogram. {latency: count} where "latency" is the
# latency in microseconds with only 2 significant figures and "count" is the
# number of response times that fell in that latency range.
latency_hist = netperf.ParseHistogram(stdout)
hist_metadata = {'histogram': json.dumps(latency_hist)}
hist_metadata.update(metadata)
latency_samples.append(sample.Sample(
'%s_Latency_Histogram' % benchmark_name, 0, 'us', hist_metadata))
if unit != MBPS:
for metric_key, metric_name in [
('50th Percentile Latency Microseconds', 'p50'),
('90th Percentile Latency Microseconds', 'p90'),
('99th Percentile Latency Microseconds', 'p99'),
('Minimum Latency Microseconds', 'min'),
('Maximum Latency Microseconds', 'max'),
('Stddev Latency Microseconds', 'stddev')]:
if metric_key in results:
latency_samples.append(
sample.Sample('%s_Latency_%s' % (benchmark_name, metric_name),
float(results[metric_key]), 'us', metadata))
return (throughput_sample, latency_samples, latency_hist)
def Run(redis_vms: List[_VM], load_vms: List[_VM],
redis_port: int) -> List[sample.Sample]:
"""Run memtier against enterprise redis and measure latency and throughput.
This function runs memtier against the redis server vm with increasing memtier
threads until one of following conditions is reached:
- FLAGS.enterprise_redis_max_threads is reached
- FLAGS.enterprise_redis_latency_threshold is reached
Args:
redis_vms: Redis server vms.
load_vms: Memtier load vms.
redis_port: Port for the redis server.
Returns:
A list of sample.Sample objects.
"""
results = []
cur_max_latency = 0.0
latency_threshold = _LATENCY_THRESHOLD.value
threads = _MIN_THREADS.value
max_threads = _MAX_THREADS.value
max_throughput_for_completion_latency_under_1ms = 0.0
redis_vm = redis_vms[0]
while (cur_max_latency < latency_threshold and threads <= max_threads):
load_command = _BuildRunCommand(redis_vms, threads, redis_port)
# 1min for throughput to stabilize and 10sec of data.
measurement_command = (
'sleep 15 && curl -v -k -u {user}:{password} '
'https://localhost:9443/v1/bdbs/stats?interval=1sec > ~/output'.
format(
user=_USERNAME,
password=FLAGS.run_uri,
))
args = [((load_vm, load_command), {}) for load_vm in load_vms]
args += [((redis_vm, measurement_command), {})]
vm_util.RunThreaded(lambda vm, command: vm.RemoteCommand(command),
args)
stdout, _ = redis_vm.RemoteCommand('cat ~/output')
output = json.loads(stdout)[0]
intervals = output.get('intervals')
for interval in intervals:
throughput = interval.get('total_req')
latency = interval.get('avg_latency')
cur_max_latency = max(cur_max_latency, latency)
sample_metadata = interval
sample_metadata['redis_tune_on_startup'] = (_TUNE_ON_STARTUP.value)
sample_metadata['redis_pipeline'] = (_PIPELINES.value)
sample_metadata['threads'] = threads
sample_metadata['shard_count'] = _SHARDS.value
sample_metadata['total_shard_count'] = _SHARDS.value * len(
redis_vms)
sample_metadata['redis_proxy_threads'] = (_PROXY_THREADS.value)
sample_metadata['redis_loadgen_clients'] = (_LOADGEN_CLIENTS.value)
sample_metadata['pin_workers'] = _PIN_WORKERS.value
sample_metadata['disable_cpus'] = _DISABLE_CPU_IDS.value
sample_metadata['redis_enterprise_version'] = _VERSION
sample_metadata['memtier_data_size'] = _DATA_SIZE.value
sample_metadata['memtier_key_maximum'] = _LOAD_RECORDS.value
results.append(
sample.Sample('throughput', throughput, 'ops/s',
sample_metadata))
if latency < 1000:
max_throughput_for_completion_latency_under_1ms = max(
max_throughput_for_completion_latency_under_1ms,
throughput)
logging.info('Threads : %d (%f, %f) < %f', threads, throughput,
latency, latency_threshold)
threads += _THREAD_INCREMENT.value
if cur_max_latency >= 1000:
results.append(
sample.Sample('max_throughput_for_completion_latency_under_1ms',
max_throughput_for_completion_latency_under_1ms,
'ops/s', sample_metadata))
return results
def ParseResults(memtier_results, meta):
"""Parse memtier_benchmark result textfile into samples.
Args:
memtier_results: Text output of running Memtier benchmark.
meta: metadata associated with the results.
Yields:
List of sample.Sample objects.
Example memtier_benchmark output, note Hits/sec and Misses/sec are displayed
in error for version 1.2.8+ due to bug:
https://github.com/RedisLabs/memtier_benchmark/issues/46
4 Threads
50 Connections per thread
20 Seconds
Type Ops/sec Hits/sec Misses/sec Latency KB/sec
------------------------------------------------------------------------
Sets 4005.50 --- --- 4.50600 308.00
Gets 40001.05 0.00 40001.05 4.54300 1519.00
Totals 44006.55 0.00 40001.05 4.54000 1828.00
Request Latency Distribution
Type <= msec Percent
------------------------------------------------------------------------
SET 0 9.33
SET 1 71.07
...
SET 33 100.00
SET 36 100.00
---
GET 0 10.09
GET 1 70.88
..
GET 40 100.00
GET 41 100.00
"""
set_histogram = []
get_histogram = []
total_requests = FLAGS.memtier_requests
approx_total_sets = round(
float(total_requests) / (FLAGS.memtier_ratio + 1))
last_total_sets = 0
approx_total_gets = total_requests - approx_total_sets
last_total_gets = 0
for raw_line in memtier_results.splitlines():
line = raw_line.strip()
if re.match(r'^Totals', line):
_, ops, _, _, _, kilobyte = line.split()
yield sample.Sample('Ops Throughput', float(ops), 'ops/s', meta)
yield sample.Sample('KB Throughput', float(kilobyte), 'KB/s', meta)
last_total_sets = _ParseLine(r'^SET', line, approx_total_sets,
last_total_sets, set_histogram)
last_total_gets = _ParseLine(r'^GET', line, approx_total_gets,
last_total_gets, get_histogram)
for name, histogram in [('get', get_histogram), ('set', set_histogram)]:
hist_meta = meta.copy()
hist_meta.update({'histogram': json.dumps(histogram)})
yield sample.Sample('{0} latency histogram'.format(name), 0, '',
hist_meta)
def ParseResults(results):
"""Result parser for SciMark2.
Sample Results (C version):
** **
** SciMark2 Numeric Benchmark, see http://math.nist.gov/scimark **
** for details. (Results can be submitted to [email protected]) **
** **
Using 2.00 seconds min time per kenel.
Composite Score: 1596.04
FFT Mflops: 1568.64 (N=1024)
SOR Mflops: 1039.98 (100 x 100)
MonteCarlo: Mflops: 497.64
Sparse matmult Mflops: 1974.39 (N=1000, nz=5000)
LU Mflops: 2899.56 (M=100, N=100)
(Yes, "kenel" is part of the original output.)
Sample Results (Java version):
SciMark 2.0a
Composite Score: 1731.4467627163242
FFT (1024): 996.9938397943672
SOR (100x100): 1333.5328291027124
Monte Carlo : 724.5221517116782
Sparse matmult (N=1000, nz=5000): 1488.18620413327
LU (100x100): 4113.998788839592
java.vendor: Oracle Corporation
java.version: 1.7.0_75
os.arch: amd64
os.name: Linux
os.version: 3.16.0-25-generic
Args:
results: SciMark2 result.
Returns:
A list of sample.Sample objects.
"""
result_start_regex = re.compile(
r"""
^
;;; \s+ (\S+) #1: Language ("C" or "Java")
\s+ (\S+) #2: Size ("small" or "large")
""", re.VERBOSE | re.MULTILINE)
score_regex = re.compile(
r"""
^ (Composite \s+ Score) : \s+ (\d+ \. \d+)
""", re.VERBOSE | re.MULTILINE)
result_regex_c = re.compile(
r"""
^
( .+? ) \s+ #1: Test name
Mflops: \s+
( \d+ \. \d+ ) #2: Test score
( \s+ \( .+? \) )? #3: Optional test details
""", re.VERBOSE | re.MULTILINE)
result_regex_java = re.compile(
r"""
^
( .+? ) #1: Test name
: \s+
( \d+ \. \d+ ) #2: Test score
""", re.VERBOSE | re.MULTILINE)
platform_regex = re.compile(
r"""
^
( \w+ \. \w+ ) #1: Property name
: \s+
( .* ) #2: Property value
""", re.VERBOSE | re.MULTILINE)
def FindBenchStart(results, start_index=0):
m = result_start_regex.search(results, start_index)
if m is None:
return -1, None, None
return m.start(), m.group(1), m.group(2)
def ExtractPlatform(result, benchmark_language):
"""Retrieves platform data from the result string."""
metadata = {}
meta_start = None
if benchmark_language == 'C':
pass
elif benchmark_language == 'Java':
for m in platform_regex.finditer(result):
if meta_start is None:
meta_start = m.start()
metadata[m.group(1)] = m.group(2)
return metadata, meta_start
def ExtractScore(result):
m = score_regex.search(result)
if m is None:
raise errors.Benchmarks.RunError(
'scimark2: Cannot find score in output.')
label = m.group(1)
score = float(m.group(2))
return score, label, m.end()
def ExtractResults(result, benchmark_language):
"""Retrieves data points from the result string."""
datapoints = []
if benchmark_language == 'C':
for groups in regex_util.ExtractAllMatches(result_regex_c, result):
metric = '{0} {1}'.format(groups[0].strip(), groups[2].strip())
metric = metric.strip().strip(
':') # Extra ':' in 'MonteCarlo:'.
value = float(groups[1])
datapoints.append((metric, value))
elif benchmark_language == 'Java':
for groups in regex_util.ExtractAllMatches(result_regex_java,
result):
datapoints.append((groups[0].strip(), float(groups[1])))
return datapoints
# Find start positions for all the test results.
tests = []
test_start_pos = 0
while True:
start_index, benchmark_language, benchmark_size = FindBenchStart(
results, test_start_pos)
if start_index == -1:
break
tests.append((start_index, benchmark_language, benchmark_size))
test_start_pos = start_index + 1
# Now loop over individual tests collecting samples.
samples = []
for test_num, test_details in enumerate(tests):
start_index, benchmark_language, benchmark_size = test_details
# Get end index - either start of next test, or None for the last test.
end_index = None
if test_num + 1 < len(tests):
end_index = tests[test_num + 1][0]
result = results[start_index:end_index]
metadata = {
'benchmark_language': benchmark_language,
'benchmark_size': benchmark_size
}
# Assume that the result consists of overall score followed by
# specific scores and then platform metadata.
# Get the metadata first since we need that to annotate samples.
platform_metadata, meta_start = ExtractPlatform(
result, benchmark_language)
metadata.update(platform_metadata)
# Get the overall score.
score, label, score_end = ExtractScore(result)
samples.append(sample.Sample(label, score, 'Mflops', metadata))
# For the specific scores, only look at the part of the string
# bounded by score_end and meta_start to avoid adding extraneous
# items. The overall score and platform data would match the
# result regex.
datapoints = ExtractResults(result[score_end:meta_start],
benchmark_language)
for metric, value in datapoints:
samples.append(sample.Sample(metric, value, 'Mflops', metadata))
return samples
def _GetCpuUtilizationSample(samples, instance_id):
"""Gets a list of cpu utilization samples - one per cluster.
Note that the utilization only covers the run stage.
Args:
samples: list of sample.Sample. Used to find the load and run samples for
computing the run time.
instance_id: the bigtable instance id.
Returns:
a sample describing the runtime
Raises:
Exception: if the time for running can not be found or if
querying the cpu sampling fails.
"""
load_sample = MaxWithDefault(
(cur_sample for cur_sample in samples
if cur_sample.metadata.get('stage') == 'load'),
key=lambda sample: sample.timestamp,
default=None)
# get the last sample recorded in the run stage
last_run_sample = MaxWithDefault(
(cur_sample for cur_sample in samples
if cur_sample.metadata.get('stage') == 'run'),
key=lambda sample: sample.timestamp,
default=None)
if not load_sample or not last_run_sample:
raise Exception('Could not find the load or run sample, '
'so cant get the time for cpu utilization')
# pylint: disable=g-import-not-at-top
from google.cloud import monitoring_v3
from google.cloud.monitoring_v3 import query
# Query the cpu utilization, which are gauged values at each minute in the
# time window.
client = monitoring_v3.MetricServiceClient()
start_timestamp = load_sample.timestamp
end_timestamp = last_run_sample.timestamp
samples = []
for metric in ['cpu_load', 'cpu_load_hottest_node']:
cpu_query = query.Query(
client,
project=(FLAGS.project or _GetDefaultProject()),
metric_type='bigtable.googleapis.com/cluster/{}'.format(metric),
end_time=datetime.datetime.utcfromtimestamp(end_timestamp),
minutes=int((end_timestamp - start_timestamp) / 60))
cpu_query = cpu_query.select_resources(instance=instance_id)
time_series = list(cpu_query)
if not time_series:
raise Exception(
'Time series for computing {} could not be found.'.format(
metric))
# Build the dict to be added to samples.
for cluster_number, cluster_time_series in enumerate(time_series):
utilization = [
round(point.value.double_value, 3)
for point in cluster_time_series.points
]
metadata = {
'cluster_number': cluster_number,
'cpu_utilization_per_minute': utilization,
}
cpu_utilization_sample = sample.Sample('{}_array'.format(metric),
-1, metric, metadata)
samples.append(cpu_utilization_sample)
return samples
GOOD_METADATA = {
'runspec_config': 'linux64-x64-gcc47.cfg',
'runspec_define': '',
'runspec_iterations': '3',
'runspec_enable_32bit': 'False',
'runspec_metric': 'rate',
'spec_runmode': 'base',
'spec17_fdo': False,
'spec17_copies': None,
'spec17_threads': None,
'spec17_subset': ['intspeed', 'fpspeed', 'intrate', 'fprate']
}
EXPECTED_RESULT_SPECINT = [
sample.Sample(metric='400.perlbench',
value=23.4,
unit='',
metadata=GOOD_METADATA),
sample.Sample(metric='401.bzip2',
value=17.1,
unit='',
metadata=GOOD_METADATA),
sample.Sample(metric='403.gcc',
value=22.1,
unit='',
metadata=GOOD_METADATA),
sample.Sample(metric='429.mcf',
value=25.1,
unit='',
metadata=GOOD_METADATA),
sample.Sample(metric='445.gobmk',
value=21.0,
def testNoneValueShouldBeZero(self):
instance = sample.Sample(metric='Test', value=None, unit='Mbps')
self.assertIsInstance(instance.value, float)
self.assertEqual(0.0, instance.value)
def _PrepareSysbench(client_vm, benchmark_spec):
"""Prepare the Sysbench OLTP test with data loading stage.
Args:
client_vm: The client VM that will issue the sysbench test.
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
Returns:
results: A list of results of the data loading step.
"""
_InstallLuaScriptsIfNecessary(client_vm)
results = []
db = benchmark_spec.relational_db
# every cloud provider has a different mechansim for setting root password
# on initialize. GCP doesnt support setting password at creation though.
# So initialize root password here early as possible.
if FLAGS.cloud == 'GCP' and FLAGS.use_managed_db:
set_db_root_password_command = 'mysqladmin -h %s -u root password %s' % (
db.endpoint, db.spec.database_password)
stdout, stderr = client_vm.RemoteCommand(set_db_root_password_command)
logging.info('Root password is set to %s.', db.spec.database_password)
# Create the sbtest database for Sysbench.
create_sbtest_db_cmd = ('mysql %s '
'-e \'create database sbtest;\'') % (
db.MakeMysqlConnectionString())
stdout, stderr = client_vm.RemoteCommand(create_sbtest_db_cmd)
logging.info('sbtest db created, stdout is %s, stderr is %s', stdout,
stderr)
# Provision the Sysbench test based on the input flags (load data into DB)
# Could take a long time if the data to be loaded is large.
data_load_start_time = time.time()
# Data loading is write only so need num_threads less than or equal to the
# amount of tables - capped at 64 threads for when number of tables
# gets very large.
num_threads = min(FLAGS.sysbench_tables, 64)
data_load_cmd_tokens = [
'nice', # run with a niceness of lower priority
'-15', # to encourage cpu time for ssh commands
'sysbench',
FLAGS.sysbench_testname,
'--tables=%d' % FLAGS.sysbench_tables,
('--table_size=%d' %
FLAGS.sysbench_table_size if _IsValidFlag('table_size') else ''),
('--scale=%d' % FLAGS.sysbench_scale if _IsValidFlag('scale') else ''),
'--threads=%d' % num_threads
]
data_load_cmd = ' '.join(data_load_cmd_tokens +
_GetCommonSysbenchOptions(benchmark_spec) +
['prepare'])
# Sysbench output is in stdout, but we also get stderr just in case
# something went wrong.
stdout, stderr = client_vm.RobustRemoteCommand(data_load_cmd)
load_duration = time.time() - data_load_start_time
logging.info('It took %d seconds to finish the data loading step',
load_duration)
logging.info('data loading results: \n stdout is:\n%s\nstderr is\n%s',
stdout, stderr)
db.mysql_db_size_MB = _GetDatabaseSize(client_vm, benchmark_spec)
metadata = CreateMetadataFromFlags(db)
results.append(
sample.Sample('sysbench data load time', load_duration, SECONDS_UNIT,
metadata))
return results
def _FailoverWorkloadThread(vm, benchmark_spec, sysbench_thread_count, results,
metadata):
"""Entry point for thraed that drives the workload for failover test.
The failover test requires 2 threads of execution. The failover thread
will cause a failover after a certain timeout.
The workload thread is responsible for driving the database while waiting
for the failover to occur. After failover, t then measures how long
it takes for the database to become operable again. Finally, the
database is run under the regular workload again to gather statistics
on how the database behaves post-failover.
Args:
vm: The vm of the client driving the database
benchmark_spec: Benchmark spec including the database spec
sysbench_thread_count: How many client threads to drive the database with
while waiting for failover
results: A list to append any sample to
metadata: Metadata to be used in sample construction
Returns:
True if the thread made it through the expected workflow without timeouts
False if database failover could not be detected or it timed out.
"""
logging.info('\n Running sysbench to drive database while '
'waiting for failover \n')
stopwatch = _Stopwatch()
if not _WaitForWorkloadToFail(stopwatch, vm, benchmark_spec,
sysbench_thread_count):
return False
if not _WaitForPingToFail(stopwatch, vm, benchmark_spec):
return False
time_until_failover = stopwatch.elapsed_seconds
results.append(
sample.Sample('time_until_failover', time_until_failover, 's',
metadata))
if not _WaitForPingToSucceed(stopwatch, vm, benchmark_spec):
return False
time_failover_to_ping = stopwatch.elapsed_seconds - time_until_failover
results.append(
sample.Sample('time_failover_to_ping', time_failover_to_ping, 's',
metadata))
if not _WaitUntilSysbenchWorkloadConnect(stopwatch, vm, benchmark_spec,
sysbench_thread_count):
return False
time_failover_to_connect = stopwatch.elapsed_seconds - time_until_failover
results.append(
sample.Sample('time_failover_to_connect', time_failover_to_connect,
's', metadata))
_GatherPostFailoverTPS(vm, benchmark_spec, sysbench_thread_count, results,
metadata)
return True
def ParseRunAct(out):
"""Parse act output.
Raw output format:
trans device
%>(ms) %>(ms)
slice 1 8 64 1 8 64
----- ------ ------ ------ ------ ------ ------
1 1.67 0.00 0.00 1.63 0.00 0.00
2 1.38 0.00 0.00 1.32 0.00 0.00
3 1.80 0.14 0.00 1.56 0.08 0.00
4 1.43 0.00 0.00 1.39 0.00 0.00
5 1.68 0.00 0.00 1.65 0.00 0.00
6 1.37 0.00 0.00 1.33 0.00 0.00
7 1.44 0.00 0.00 1.41 0.00 0.00
8 1.41 0.00 0.00 1.35 0.00 0.00
9 2.70 0.73 0.00 1.91 0.08 0.00
10 1.54 0.00 0.00 1.51 0.00 0.00
11 1.53 0.00 0.00 1.48 0.00 0.00
12 1.47 0.00 0.00 1.43 0.00 0.00
----- ------ ------ ------ ------ ------ ------
avg 1.62 0.07 0.00 1.50 0.01 0.00
max 2.70 0.73 0.00 1.91 0.08 0.00
Args:
out: string. Output from act test.
Returns:
A list of sample.Sample objects.
"""
lines = out.split('\n')
buckets = []
ret = []
for line in lines:
vals = line.split()
if not vals or '-' in vals[0]:
continue
if vals[0] == 'slice':
for v in vals[1:]:
buckets.append(int(v))
continue
if not buckets:
continue
matrix = ''
if vals[0] in ('avg', 'max'):
matrix = '_' + vals[0]
num_buckets = (len(vals) - 1) / 2
for i in xrange(num_buckets):
assert buckets[i] == buckets[i + num_buckets]
ret.append(
sample.Sample('trans' + matrix, float(vals[i + 1]), '%>(ms)', {
'slice': vals[0],
'bucket': buckets[i]
}))
ret.append(
sample.Sample('device' + matrix,
float(vals[i + num_buckets + 1]), '%>(ms)', {
'slice': vals[0],
'bucket': buckets[i + num_buckets]
}))
return ret
def testParseCSVResults(self):
result = bonnie_benchmark.ParseCSVResults(self.contents)
expected_result = [[
'put_block', 72853.0, 'K/sec', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'put_block_cpu', 15.0, '%s', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'rewrite', 47358.0, 'K/sec', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'rewrite_cpu', 5.0, '%s', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'get_block', 156821.0, 'K/sec', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'get_block_cpu', 7.0, '%s', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'seeks', 537.7, 'K/sec', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'seeks_cpu', 10.0, '%s', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'seq_create', 49223.0, 'K/sec', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'seq_create_cpu', 58.0, '%s', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'seq_del', 54405.0, 'K/sec', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'seq_del_cpu', 53.0, '%s', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'ran_create', 2898.0, 'K/sec', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'ran_create_cpu', 97.0, '%s', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'ran_del', 59089.0, 'K/sec', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'ran_del_cpu', 60.0, '%s', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'put_block_latency', 512.0, 'ms', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'rewrite_latency', 670.0, 'ms', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'get_block_latency', 44660.0, 'us', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'seeks_latency', 200.0, 'ms', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'seq_create_latency', 3747.0, 'us', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'seq_stat_latency', 1759.0, 'us', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'seq_del_latency', 1643.0, 'us', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'ran_create_latency', 33518.0, 'us', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'ran_stat_latency', 192.0, 'us', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
],
[
'ran_del_latency', 839.0, 'us', {
'name': 'perfkit-7b22f510-0',
'format_version': '1.96',
'num_files': '100',
'seed': '1421800799',
'concurrency': '1',
'file_size': '7423M',
'bonnie_version': '1.96'
}
]]
expected_result = [
sample.Sample(*sample_tuple) for sample_tuple in expected_result
]
self.assertSampleListsEqualUpToTimestamp(result, expected_result)
def _GetCpuUtilizationSample(samples: List[sample.Sample],
instance_id: str) -> List[sample.Sample]:
"""Gets a list of cpu utilization samples - one per cluster per workload.
Note that the utilization only covers the workload run stage.
Args:
samples: list of sample.Sample. Used to find the timestamp information to
determine the time windows for the cpu metrics.
instance_id: the bigtable instance id.
Returns:
a list of samples for metrics "cpu_load" and "cpu_load_hottest_node",
"""
runtime_samples = [
s for s in samples
if s.metadata.get('stage') == 'run' and s.metric == 'overall RunTime'
]
# pylint: disable=g-import-not-at-top
from google.cloud import monitoring_v3
from google.cloud.monitoring_v3 import query
from google.cloud.monitoring_v3.gapic.transports import metric_service_grpc_transport
client = monitoring_v3.MetricServiceClient(
transport=metric_service_grpc_transport.MetricServiceGrpcTransport(
address=_MONITORING_ADDRESS.value))
cpu_samples = []
time_units_in_secs = {'s': 1, 'ms': 0.001, 'us': 0.000001}
for runtime_sample in runtime_samples:
if runtime_sample.unit not in time_units_in_secs:
logging.warning('The unit of overall RunTime is not supported: %s',
runtime_sample.unit)
continue
duration_sec = runtime_sample.value * time_units_in_secs.get(
runtime_sample.unit)
workload_duration_minutes = max(1, int(duration_sec / 60))
# workload_index helps associate the cpu metrics with the current run stage.
workload_index = runtime_sample.metadata.get('workload_index')
# Query the cpu utilization, which are gauged values at each minute in the
# time window determined by end_timestamp and workload_duration_minutes.
end_timestamp = runtime_sample.timestamp
for metric in ['cpu_load', 'cpu_load_hottest_node']:
cpu_query = query.Query(
client, project=(FLAGS.project or _GetDefaultProject()),
metric_type=f'bigtable.googleapis.com/cluster/{metric}',
end_time=datetime.datetime.utcfromtimestamp(end_timestamp),
minutes=workload_duration_minutes)
cpu_query = cpu_query.select_resources(instance=instance_id)
time_series = list(cpu_query)
if not time_series:
logging.debug(
'Time series for computing %s could not be found.', metric)
continue
# Build and add the cpu samples from the query results.
for cluster_number, cluster_time_series in enumerate(time_series):
utilization = [
round(point.value.double_value, 3)
for point in cluster_time_series.points]
average_utilization = round(sum(utilization) / len(utilization), 3)
metadata = {
'cluster_number': cluster_number,
'workload_index': workload_index,
'cpu_utilization_per_minute': utilization,
'cpu_average_utilization': average_utilization,
}
cpu_utilization_sample = sample.Sample(
f'{metric}_array', -1, '', metadata)
cpu_samples.append(cpu_utilization_sample)
return cpu_samples
