def RunBenchmarks():
"""Runs all benchmarks in PerfKitBenchmarker.
Returns:
Exit status for the process.
"""
benchmark_specs = _CreateBenchmarkSpecs()
if FLAGS.randomize_run_order:
random.shuffle(benchmark_specs)
if FLAGS.dry_run:
print 'PKB will run with the following configurations:'
for spec in benchmark_specs:
print spec
print ''
return 0
collector = SampleCollector()
try:
tasks = [(RunBenchmarkTask, (spec,), {})
for spec in benchmark_specs]
if FLAGS.run_with_pdb and FLAGS.run_processes == 1:
spec_sample_tuples = RunBenchmarkTasksInSeries(tasks)
else:
spec_sample_tuples = background_tasks.RunParallelProcesses(
tasks, FLAGS.run_processes, FLAGS.run_processes_delay)
benchmark_specs, sample_lists = zip(*spec_sample_tuples)
for sample_list in sample_lists:
collector.samples.extend(sample_list)
finally:
if collector.samples:
collector.PublishSamples()
if benchmark_specs:
logging.info(benchmark_status.CreateSummary(benchmark_specs))
logging.info('Complete logs can be found at: %s',
vm_util.PrependTempDir(LOG_FILE_NAME))
logging.info('Completion statuses can be found at: %s',
vm_util.PrependTempDir(COMPLETION_STATUS_FILE_NAME))
if stages.TEARDOWN not in FLAGS.run_stage:
logging.info(
'To run again with this setup, please use --run_uri=%s', FLAGS.run_uri)
if FLAGS.archive_bucket:
archive.ArchiveRun(vm_util.GetTempDir(), FLAGS.archive_bucket,
gsutil_path=FLAGS.gsutil_path,
prefix=FLAGS.run_uri + '_')
# Write completion status file(s)
completion_status_file_name = (
vm_util.PrependTempDir(COMPLETION_STATUS_FILE_NAME))
with open(completion_status_file_name, 'w') as status_file:
_WriteCompletionStatusFile(benchmark_specs, status_file)
if FLAGS.completion_status_file:
with open(FLAGS.completion_status_file, 'w') as status_file:
_WriteCompletionStatusFile(benchmark_specs, status_file)
all_benchmarks_succeeded = all(spec.status == benchmark_status.SUCCEEDED
for spec in benchmark_specs)
return 0 if all_benchmarks_succeeded else 1
def ParseResults(job_file,
fio_json_result,
base_metadata=None,
log_file_base='',
bin_vals=None,
skip_latency_individual_stats=False,
instance=None):
"""Parse fio json output into samples.
Args:
job_file: The contents of the fio job file.
fio_json_result: Fio results in json format.
base_metadata: Extra metadata to annotate the samples with.
log_file_base: String. Base name for fio log files.
bin_vals: A 2-D list of int. Each list represents a list of
bin values in histgram log. Calculated from remote VM using
fio/tools/hist/fiologparser_hist.py
skip_latency_individual_stats: Bool. If true, skips pulling latency stats
that are not aggregate.
instance: on which machines the test has been executed
Returns:
A list of sample.Sample objects.
"""
samples = []
# The samples should all have the same timestamp because they
# come from the same fio run.
timestamp = time.time()
parameter_metadata = ParseJobFile(job_file)
io_modes = list(DATA_DIRECTION.values())
# clat_hist files are indexed sequentially by inner job. If you have a job
# file with 2 jobs, each with numjobs=4 you will have 8 clat_hist files.
clat_hist_idx = 0
for job in fio_json_result['jobs']:
job_name = job['jobname']
parameters = parameter_metadata[job_name]
parameters['fio_job'] = job_name
if base_metadata:
parameters.update(base_metadata)
for mode in io_modes:
if job[mode]['io_bytes']:
metric_name = '%s:%s' % (job_name, mode)
bw_metadata = {
'bw_min': job[mode]['bw_min'],
'bw_max': job[mode]['bw_max'],
'bw_dev': job[mode]['bw_dev'],
'bw_agg': job[mode]['bw_agg'],
'bw_mean': job[mode]['bw_mean']
}
bw_metadata.update(parameters)
samples.append(
sample.Sample('%s:bandwidth' % metric_name,
job[mode]['bw'],
'KB/s',
bw_metadata,
instance=instance))
# There is one sample whose metric is '<metric_name>:latency'
# with all of the latency statistics in its metadata, and then
# a bunch of samples whose metrics are
# '<metric_name>:latency:min' through
# '<metric_name>:latency:p99.99' that hold the individual
# latency numbers as values. This is for historical reasons.
clat_key = 'clat' if 'clat' in job[mode] else 'clat_ns'
clat_section = job[mode][clat_key]
def _ConvertClat(value):
if clat_key is 'clat_ns':
# convert from nsec to usec
return value / 1000
else:
return value
percentiles = clat_section['percentile']
lat_statistics = [('min', _ConvertClat(clat_section['min'])),
('max', _ConvertClat(clat_section['max'])),
('mean', _ConvertClat(clat_section['mean'])),
('stddev',
_ConvertClat(clat_section['stddev']))]
if not skip_latency_individual_stats:
lat_statistics += [
('p1', _ConvertClat(percentiles['1.000000'])),
('p5', _ConvertClat(percentiles['5.000000'])),
('p10', _ConvertClat(percentiles['10.000000'])),
('p20', _ConvertClat(percentiles['20.000000'])),
('p30', _ConvertClat(percentiles['30.000000'])),
('p40', _ConvertClat(percentiles['40.000000'])),
('p50', _ConvertClat(percentiles['50.000000'])),
('p60', _ConvertClat(percentiles['60.000000'])),
('p70', _ConvertClat(percentiles['70.000000'])),
('p80', _ConvertClat(percentiles['80.000000'])),
('p90', _ConvertClat(percentiles['90.000000'])),
('p95', _ConvertClat(percentiles['95.000000'])),
('p99', _ConvertClat(percentiles['99.000000'])),
('p99.5', _ConvertClat(percentiles['99.500000'])),
('p99.9', _ConvertClat(percentiles['99.900000'])),
('p99.95', _ConvertClat(percentiles['99.950000'])),
('p99.99', _ConvertClat(percentiles['99.990000']))
]
lat_metadata = parameters.copy()
for name, val in lat_statistics:
lat_metadata[name] = val
samples.append(
sample.Sample('%s:latency' % metric_name,
_ConvertClat(job[mode][clat_key]['mean']),
'usec',
lat_metadata,
timestamp,
instance=instance))
for stat_name, stat_val in lat_statistics:
samples.append(
sample.Sample('%s:latency:%s' %
(metric_name, stat_name),
stat_val,
'usec',
parameters,
timestamp,
instance=instance))
samples.append(
sample.Sample('%s:iops' % metric_name,
job[mode]['iops'],
'',
parameters,
timestamp,
instance=instance))
if log_file_base and bin_vals:
# Parse histograms
aggregates = collections.defaultdict(collections.Counter)
for _ in range(int(parameters.get('numjobs', 1))):
clat_hist_idx += 1
hist_file_path = vm_util.PrependTempDir(
'%s_clat_hist.%s.log' %
(log_file_base, str(clat_hist_idx)))
hists = _ParseHistogram(hist_file_path,
bin_vals[clat_hist_idx - 1])
for key in hists:
aggregates[key].update(hists[key])
samples += _BuildHistogramSamples(aggregates,
job_name,
parameters,
instance=instance)
return samples
def SetUpPKB():
"""Set globals and environment variables for PKB.
After SetUpPKB() returns, it should be possible to call PKB
functions, like benchmark_spec.Prepare() or benchmark_spec.Run().
SetUpPKB() also modifies the local file system by creating a temp
directory and storing new SSH keys.
"""
try:
_InitializeRunUri()
except errors.Error as e:
logging.error(e)
sys.exit(1)
# Initialize logging.
vm_util.GenTempDir()
if FLAGS.use_pkb_logging:
log_util.ConfigureLogging(
stderr_log_level=log_util.LOG_LEVELS[FLAGS.log_level],
log_path=vm_util.PrependTempDir(LOG_FILE_NAME),
run_uri=FLAGS.run_uri,
file_log_level=log_util.LOG_LEVELS[FLAGS.file_log_level])
logging.info('PerfKitBenchmarker version: %s', version.VERSION)
# Translate deprecated flags and log all provided flag values.
disk.WarnAndTranslateDiskFlags()
_LogCommandLineFlags()
# Register skip pending runs functionality.
RegisterSkipPendingRunsCheck(_SkipPendingRunsFile)
# Check environment.
if not FLAGS.ignore_package_requirements:
requirements.CheckBasicRequirements()
for executable in REQUIRED_EXECUTABLES:
if not vm_util.ExecutableOnPath(executable):
raise errors.Setup.MissingExecutableError(
'Could not find required executable "%s"' % executable)
# Check mutually exclusive flags
if FLAGS.run_stage_iterations > 1 and FLAGS.run_stage_time > 0:
raise errors.Setup.InvalidFlagConfigurationError(
'Flags run_stage_iterations and run_stage_time are mutually exclusive'
)
vm_util.SSHKeyGen()
if FLAGS.static_vm_file:
with open(FLAGS.static_vm_file) as fp:
static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile(
fp)
events.initialization_complete.send(parsed_flags=FLAGS)
benchmark_lookup.SetBenchmarkModuleFunction(benchmark_sets.BenchmarkModule)
package_lookup.SetPackageModuleFunction(benchmark_sets.PackageModule)
# Update max_concurrent_threads to use at least as many threads as VMs. This
# is important for the cluster_boot benchmark where we want to launch the VMs
# in parallel.
if not FLAGS.max_concurrent_threads:
FLAGS.max_concurrent_threads = max(
background_tasks.MAX_CONCURRENT_THREADS, FLAGS.num_vms)
logging.info('Setting --max_concurrent_threads=%d.',
FLAGS.max_concurrent_threads)
def RunWithExec(vm, exec_path, remote_job_file_path, job_file_contents):
"""Spawn fio and gather the results.
Args:
vm: vm to run the benchmark on.
exec_path: string path to the fio executable.
remote_job_file_path: path, on the vm, to the location of the job file.
job_file_contents: string contents of the fio job file.
Returns:
A list of sample.Sample objects.
"""
logging.info('FIO running on %s', vm)
disk = vm.scratch_disks[0]
mount_point = disk.mount_point
if FLAGS.fio_write_against_multiple_clients:
mount_point = '%s/%s' % (disk.mount_point, vm.name)
logging.info('FIO mount point changed to %s', mount_point)
job_file_string = GetOrGenerateJobFileString(
FLAGS.fio_jobfile, FLAGS.fio_generate_scenarios, AgainstDevice(), disk,
FLAGS.fio_io_depths, FLAGS.fio_num_jobs, FLAGS.fio_working_set_size,
FLAGS.fio_blocksize, FLAGS.fio_runtime, FLAGS.fio_parameters,
job_file_contents)
job_file_path = vm_util.PrependTempDir(vm.name + LOCAL_JOB_FILE_SUFFIX)
with open(job_file_path, 'w') as job_file:
job_file.write(job_file_string)
logging.info('Wrote fio job file at %s', job_file_path)
logging.info(job_file_string)
vm.PushFile(job_file_path, remote_job_file_path)
if AgainstDevice():
fio_command = '%s --output-format=json --filename=%s %s' % (
exec_path, disk.GetDevicePath(), remote_job_file_path)
else:
fio_command = '%s --output-format=json --directory=%s %s' % (
exec_path, mount_point, remote_job_file_path)
collect_logs = any([
FLAGS.fio_lat_log, FLAGS.fio_bw_log, FLAGS.fio_iops_log,
FLAGS.fio_hist_log
])
log_file_base = ''
if collect_logs:
log_file_base = '%s_%s' % (PKB_FIO_LOG_FILE_NAME, str(time.time()))
fio_command = ' '.join([fio_command, GetLogFlags(log_file_base)])
# TODO(user): This only gives results at the end of a job run
# so the program pauses here with no feedback to the user.
# This is a pretty lousy experience.
logging.info('FIO Results:')
start_time = time.time()
stdout, _ = vm.RobustRemoteCommand(fio_command, should_log=True)
end_time = time.time()
bin_vals = []
if collect_logs:
vm.PullFile(vm_util.GetTempDir(), '%s*.log' % log_file_base)
if FLAGS.fio_hist_log:
num_logs = int(
vm.RemoteCommand('ls %s_clat_hist.*.log | wc -l' %
log_file_base)[0])
bin_vals += [
fio.ComputeHistogramBinVals(
vm, '%s_clat_hist.%s.log' % (log_file_base, idx + 1))
for idx in range(num_logs)
]
samples = fio.ParseResults(job_file_string,
json.loads(stdout),
log_file_base=log_file_base,
bin_vals=bin_vals)
samples.append(
sample.Sample('start_time', start_time, 'sec', samples[0].metadata))
samples.append(
sample.Sample('end_time', end_time, 'sec', samples[0].metadata))
return samples
def RunBenchmarks(publish=True):
"""Runs all benchmarks in PerfKitBenchmarker.
Args:
publish: A boolean indicating whether results should be published.
Returns:
Exit status for the process.
"""
if FLAGS.version:
print version.VERSION
return
_LogCommandLineFlags()
if FLAGS.os_type == benchmark_spec.WINDOWS and not vm_util.RunningOnWindows(
):
logging.error('In order to run benchmarks on Windows VMs, you must be '
'running on Windows.')
return 1
collector = SampleCollector()
if FLAGS.static_vm_file:
with open(FLAGS.static_vm_file) as fp:
static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile(
fp)
run_status_lists = []
benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags()
total_benchmarks = len(benchmark_tuple_list)
benchmark_counts = collections.defaultdict(itertools.count)
args = []
for i, benchmark_tuple in enumerate(benchmark_tuple_list):
benchmark_module, user_config = benchmark_tuple
benchmark_name = benchmark_module.BENCHMARK_NAME
benchmark_uid = benchmark_name + str(
benchmark_counts[benchmark_name].next())
run_status_lists.append(
[benchmark_name, benchmark_uid, benchmark_status.SKIPPED])
args.append((benchmark_module, collector, i + 1, total_benchmarks,
benchmark_module.GetConfig(user_config), benchmark_uid))
try:
for run_args, run_status_list in zip(args, run_status_lists):
benchmark_module, _, sequence_number, _, _, benchmark_uid = run_args
benchmark_name = benchmark_module.BENCHMARK_NAME
try:
run_status_list[2] = benchmark_status.FAILED
RunBenchmark(*run_args)
run_status_list[2] = benchmark_status.SUCCEEDED
except BaseException as e:
msg = 'Benchmark {0}/{1} {2} (UID: {3}) failed.'.format(
sequence_number, total_benchmarks, benchmark_name,
benchmark_uid)
if (isinstance(e, KeyboardInterrupt)
or FLAGS.stop_after_benchmark_failure):
logging.error('%s Execution will not continue.', msg)
break
else:
logging.error('%s Execution will continue.', msg)
finally:
if collector.samples:
collector.PublishSamples()
if run_status_lists:
logging.info(benchmark_status.CreateSummary(run_status_lists))
logging.info('Complete logs can be found at: %s',
vm_util.PrependTempDir(LOG_FILE_NAME))
if FLAGS.run_stage not in [STAGE_ALL, STAGE_TEARDOWN]:
logging.info('To run again with this setup, please use --run_uri=%s',
FLAGS.run_uri)
if FLAGS.archive_bucket:
archive.ArchiveRun(vm_util.GetTempDir(),
FLAGS.archive_bucket,
gsutil_path=FLAGS.gsutil_path,
prefix=FLAGS.run_uri + '_')
all_benchmarks_succeeded = all(r[2] == benchmark_status.SUCCEEDED
for r in run_status_lists)
return 0 if all_benchmarks_succeeded else 1
def Run(benchmark_spec):
"""Spawn fio and gather the results.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
Returns:
A list of sample.Sample objects.
"""
vm = benchmark_spec.vms[0]
logging.info('FIO running on %s', vm)
disk = vm.scratch_disks[0]
mount_point = disk.mount_point
job_file_string = GetOrGenerateJobFileString(
FLAGS.fio_jobfile, FLAGS.fio_generate_scenarios, AgainstDevice(), disk,
FLAGS.fio_io_depths, FLAGS.fio_num_jobs, FLAGS.fio_working_set_size,
FLAGS.fio_blocksize, FLAGS.fio_runtime, FLAGS.fio_parameters)
job_file_path = vm_util.PrependTempDir(LOCAL_JOB_FILE_NAME)
with open(job_file_path, 'w') as job_file:
job_file.write(job_file_string)
logging.info('Wrote fio job file at %s', job_file_path)
vm.PushFile(job_file_path, REMOTE_JOB_FILE_PATH)
if AgainstDevice():
fio_command = 'sudo %s --output-format=json --filename=%s %s' % (
fio.FIO_PATH, disk.GetDevicePath(), REMOTE_JOB_FILE_PATH)
else:
fio_command = 'sudo %s --output-format=json --directory=%s %s' % (
fio.FIO_PATH, mount_point, REMOTE_JOB_FILE_PATH)
collect_logs = any([
FLAGS.fio_lat_log, FLAGS.fio_bw_log, FLAGS.fio_iops_log,
FLAGS.fio_hist_log
])
log_file_base = ''
if collect_logs:
log_file_base = '%s_%s' % (PKB_FIO_LOG_FILE_NAME, str(time.time()))
fio_command = ' '.join([fio_command, GetLogFlags(log_file_base)])
# TODO(user): This only gives results at the end of a job run
# so the program pauses here with no feedback to the user.
# This is a pretty lousy experience.
logging.info('FIO Results:')
stdout, stderr = vm.RobustRemoteCommand(fio_command, should_log=True)
bin_vals = []
if collect_logs:
vm.PullFile(vm_util.GetTempDir(), '%s*.log' % log_file_base)
if FLAGS.fio_hist_log:
num_logs = int(
vm.RemoteCommand('ls %s_clat_hist.*.log | wc -l' %
log_file_base)[0])
bin_vals += [
fio.ComputeHistogramBinVals(
vm, '%s_clat_hist.%s.log' % (log_file_base, idx + 1))
for idx in range(num_logs)
]
samples = fio.ParseResults(job_file_string,
json.loads(stdout),
log_file_base=log_file_base,
bin_vals=bin_vals)
return samples
def RunBenchmarks():
"""Runs all benchmarks in PerfKitBenchmarker.
Returns:
Exit status for the process.
"""
if FLAGS.version:
print version.VERSION
return
_LogCommandLineFlags()
if FLAGS.os_type == os_types.WINDOWS and not vm_util.RunningOnWindows():
logging.error('In order to run benchmarks on Windows VMs, you must be '
'running on Windows.')
return 1
collector = SampleCollector()
if FLAGS.static_vm_file:
with open(FLAGS.static_vm_file) as fp:
static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile(
fp)
benchmark_run_list = _CreateBenchmarkRunList()
try:
for run_args, run_status_list in benchmark_run_list:
benchmark_module, sequence_number, _, _, benchmark_uid = run_args
benchmark_name = benchmark_module.BENCHMARK_NAME
try:
run_status_list[2] = benchmark_status.FAILED
RunBenchmark(*run_args, collector=collector)
run_status_list[2] = benchmark_status.SUCCEEDED
except BaseException as e:
msg = 'Benchmark {0}/{1} {2} (UID: {3}) failed.'.format(
sequence_number, len(benchmark_run_list), benchmark_name,
benchmark_uid)
if (isinstance(e, KeyboardInterrupt)
or FLAGS.stop_after_benchmark_failure):
logging.error('%s Execution will not continue.', msg)
break
else:
logging.error('%s Execution will continue.', msg)
finally:
if collector.samples:
collector.PublishSamples()
if benchmark_run_list:
run_status_lists = tuple(r for _, r in benchmark_run_list)
logging.info(benchmark_status.CreateSummary(run_status_lists))
logging.info('Complete logs can be found at: %s',
vm_util.PrependTempDir(LOG_FILE_NAME))
if stages.TEARDOWN not in FLAGS.run_stage:
logging.info('To run again with this setup, please use --run_uri=%s',
FLAGS.run_uri)
if FLAGS.archive_bucket:
archive.ArchiveRun(vm_util.GetTempDir(),
FLAGS.archive_bucket,
gsutil_path=FLAGS.gsutil_path,
prefix=FLAGS.run_uri + '_')
all_benchmarks_succeeded = all(r[2] == benchmark_status.SUCCEEDED
for _, r in benchmark_run_list)
return 0 if all_benchmarks_succeeded else 1
def RunBenchmarks(publish=True):
"""Runs all benchmarks in PerfKitBenchmarker.
Args:
publish: A boolean indicating whether results should be published.
Returns:
Exit status for the process.
"""
if FLAGS.version:
print version.VERSION
return
for executable in REQUIRED_EXECUTABLES:
if not vm_util.ExecutableOnPath(executable):
logging.error('Could not find required executable "%s".' % executable)
return 1
if FLAGS.run_uri is None:
if FLAGS.run_stage not in [STAGE_ALL, STAGE_PREPARE]:
# Attempt to get the last modified run directory.
run_uri = vm_util.GetLastRunUri()
if run_uri:
FLAGS.run_uri = run_uri
logging.warning(
'No run_uri specified. Attempting to run "%s" with --run_uri=%s.',
FLAGS.run_stage, FLAGS.run_uri)
else:
logging.error(
'No run_uri specified. Could not run "%s".', FLAGS.run_stage)
return 1
else:
FLAGS.run_uri = str(uuid.uuid4())[-8:]
elif not FLAGS.run_uri.isalnum() or len(FLAGS.run_uri) > MAX_RUN_URI_LENGTH:
logging.error('run_uri must be alphanumeric and less than or equal '
'to 10 characters in length.')
return 1
vm_util.GenTempDir()
log_util.ConfigureLogging(
stderr_log_level=log_util.LOG_LEVELS[FLAGS.log_level],
log_path=vm_util.PrependTempDir(LOG_FILE_NAME),
run_uri=FLAGS.run_uri)
_LogCommandLineFlags()
if FLAGS.os_type == benchmark_spec.WINDOWS and not vm_util.RunningOnWindows():
logging.error('In order to run benchmarks on Windows VMs, you must be '
'running on Windows.')
return 1
vm_util.SSHKeyGen()
collector = SampleCollector()
events.initialization_complete.send(parsed_flags=FLAGS)
if FLAGS.static_vm_file:
with open(FLAGS.static_vm_file) as fp:
static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile(
fp)
if FLAGS.benchmark_config_pair:
# Convert benchmark_config_pair into a {benchmark_name: file_name}
# dictionary.
tmp_dict = {}
for config_pair in FLAGS.benchmark_config_pair:
pair = config_pair.split(':')
tmp_dict[pair[0]] = pair[1]
FLAGS.benchmark_config_pair = tmp_dict
try:
benchmark_list = benchmark_sets.GetBenchmarksFromFlags()
total_benchmarks = len(benchmark_list)
if FLAGS.parallelism > 1:
sequence_range = range(total_benchmarks, 0, -1)
args = [((benchmark, collector, sequence_counter, total_benchmarks), {})
for benchmark, sequence_counter
in zip(benchmark_list, sequence_range)]
vm_util.RunThreaded(
RunBenchmark, args, max_concurrent_threads=FLAGS.parallelism)
else:
sequence_range = range(1, total_benchmarks + 1)
for benchmark, sequence_counter in zip(benchmark_list, sequence_range):
RunBenchmark(benchmark, collector, sequence_counter, total_benchmarks)
finally:
if collector.samples:
collector.PublishSamples()
logging.info('Complete logs can be found at: %s',
vm_util.PrependTempDir(LOG_FILE_NAME))
if FLAGS.run_stage not in [STAGE_ALL, STAGE_CLEANUP]:
logging.info(
'To run again with this setup, please use --run_uri=%s', FLAGS.run_uri)
def Run(benchmark_spec):
"""Spawn fio and gather the results.
Args:
benchmark_spec: The benchmark specification. Contains all data that is
required to run the benchmark.
Returns:
A list of sample.Sample objects.
"""
vm = benchmark_spec.vms[0]
logging.info('FIO running on %s', vm)
disk = vm.scratch_disks[0]
mount_point = disk.mount_point
job_file_string = GetOrGenerateJobFileString(FLAGS.fio_jobfile,
FLAGS.fio_generate_scenarios,
AgainstDevice(), disk,
FLAGS.fio_io_depths,
FLAGS.fio_working_set_size)
job_file_path = vm_util.PrependTempDir(LOCAL_JOB_FILE_NAME)
with open(job_file_path, 'w') as job_file:
job_file.write(job_file_string)
logging.info('Wrote fio job file at %s', job_file_path)
vm.PushFile(job_file_path, REMOTE_JOB_FILE_PATH)
if AgainstDevice():
fio_command = 'sudo %s --output-format=json --filename=%s %s' % (
fio.FIO_PATH, disk.GetDevicePath(), REMOTE_JOB_FILE_PATH)
else:
fio_command = 'sudo %s --output-format=json --directory=%s %s' % (
fio.FIO_PATH, mount_point, REMOTE_JOB_FILE_PATH)
samples = []
def RunIt(repeat_number=None,
minutes_since_start=None,
total_repeats=None):
"""Run the actual fio command on the VM and save the results.
Args:
repeat_number: if given, our number in a sequence of repetitions.
minutes_since_start: if given, minutes since the start of repetition.
total_repeats: if given, the total number of repetitions to do.
"""
if repeat_number:
logging.info('**** Repetition number %s of %s ****', repeat_number,
total_repeats)
stdout, stderr = vm.RobustRemoteCommand(fio_command, should_log=True)
if repeat_number:
base_metadata = {
'repeat_number': repeat_number,
'minutes_since_start': minutes_since_start
}
else:
base_metadata = None
samples.extend(
fio.ParseResults(job_file_string,
json.loads(stdout),
base_metadata=base_metadata))
# TODO(user): This only gives results at the end of a job run
# so the program pauses here with no feedback to the user.
# This is a pretty lousy experience.
logging.info('FIO Results:')
if not FLAGS['fio_run_for_minutes'].present:
RunIt()
else:
RunForMinutes(RunIt, FLAGS.fio_run_for_minutes, MINUTES_PER_JOB)
return samples
def ParseResults(job_file, fio_json_result, base_metadata=None,
log_file_base='', bin_vals=[]):
"""Parse fio json output into samples.
Args:
job_file: The contents of the fio job file.
fio_json_result: Fio results in json format.
base_metadata: Extra metadata to annotate the samples with.
log_file_base: String. Base name for fio log files.
bin_vals: A 2-D list of int. Each list represents a list of
bin values in histgram log. Calculated from remote VM using
fio/tools/hist/fiologparser_hist.py
Returns:
A list of sample.Sample objects.
"""
samples = []
# The samples should all have the same timestamp because they
# come from the same fio run.
timestamp = time.time()
parameter_metadata = ParseJobFile(job_file)
io_modes = DATA_DIRECTION.values()
for (idx, job) in enumerate(fio_json_result['jobs']):
job_name = job['jobname']
parameters = parameter_metadata[job_name]
parameters['fio_job'] = job_name
if base_metadata:
parameters.update(base_metadata)
for mode in io_modes:
if job[mode]['io_bytes']:
metric_name = '%s:%s' % (job_name, mode)
bw_metadata = {
'bw_min': job[mode]['bw_min'],
'bw_max': job[mode]['bw_max'],
'bw_dev': job[mode]['bw_dev'],
'bw_agg': job[mode]['bw_agg'],
'bw_mean': job[mode]['bw_mean']}
bw_metadata.update(parameters)
samples.append(
sample.Sample('%s:bandwidth' % metric_name,
job[mode]['bw'],
'KB/s', bw_metadata))
# There is one sample whose metric is '<metric_name>:latency'
# with all of the latency statistics in its metadata, and then
# a bunch of samples whose metrics are
# '<metric_name>:latency:min' through
# '<metric_name>:latency:p99.99' that hold the individual
# latency numbers as values. This is for historical reasons.
clat_section = job[mode]['clat']
percentiles = clat_section['percentile']
lat_statistics = [
('min', clat_section['min']),
('max', clat_section['max']),
('mean', clat_section['mean']),
('stddev', clat_section['stddev']),
('p1', percentiles['1.000000']),
('p5', percentiles['5.000000']),
('p10', percentiles['10.000000']),
('p20', percentiles['20.000000']),
('p30', percentiles['30.000000']),
('p40', percentiles['40.000000']),
('p50', percentiles['50.000000']),
('p60', percentiles['60.000000']),
('p70', percentiles['70.000000']),
('p80', percentiles['80.000000']),
('p90', percentiles['90.000000']),
('p95', percentiles['95.000000']),
('p99', percentiles['99.000000']),
('p99.5', percentiles['99.500000']),
('p99.9', percentiles['99.900000']),
('p99.95', percentiles['99.950000']),
('p99.99', percentiles['99.990000'])]
lat_metadata = parameters.copy()
for name, val in lat_statistics:
lat_metadata[name] = val
samples.append(
sample.Sample('%s:latency' % metric_name,
job[mode]['clat']['mean'],
'usec', lat_metadata, timestamp))
for stat_name, stat_val in lat_statistics:
samples.append(
sample.Sample('%s:latency:%s' % (metric_name, stat_name),
stat_val, 'usec', parameters, timestamp))
samples.append(
sample.Sample('%s:iops' % metric_name,
job[mode]['iops'], '', parameters, timestamp))
if log_file_base and bin_vals:
# Parse histograms
hist_file_path = vm_util.PrependTempDir(
'%s_clat_hist.%s.log' % (log_file_base, str(idx + 1)))
samples += _ParseHistogram(
hist_file_path, bin_vals[idx], job_name, parameters)
return samples
def RunBenchmarks(publish=True):
"""Runs all benchmarks in PerfKitBenchmarker.
Args:
publish: A boolean indicating whether results should be published.
Returns:
Exit status for the process.
"""
if FLAGS.version:
print version.VERSION
return
if FLAGS.run_uri is None:
if FLAGS.run_stage not in [STAGE_ALL, STAGE_PREPARE]:
logging.error('Cannot run "%s" with unspecified run_uri.',
FLAGS.run_stage)
return 1
else:
FLAGS.run_uri = str(uuid.uuid4())[-8:]
elif not FLAGS.run_uri.isalnum() or len(
FLAGS.run_uri) > MAX_RUN_URI_LENGTH:
logging.error('run_uri must be alphanumeric and less than or equal '
'to 10 characters in length.')
return 1
vm_util.GenTempDir()
log_util.ConfigureLogging(
stderr_log_level=log_util.LOG_LEVELS[FLAGS.log_level],
log_path=vm_util.PrependTempDir('pkb.log'),
run_uri=FLAGS.run_uri)
unknown_benchmarks = ListUnknownBenchmarks()
if unknown_benchmarks:
logging.error('Unknown benchmark(s) provided: %s',
', '.join(unknown_benchmarks))
return 1
vm_util.SSHKeyGen()
collector = SampleCollector()
if FLAGS.static_vm_file:
with open(FLAGS.static_vm_file) as fp:
static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile(
fp)
if FLAGS.benchmark_config_pair:
# Convert benchmark_config_pair into a {benchmark_name: file_name}
# dictionary.
tmp_dict = {}
for config_pair in FLAGS.benchmark_config_pair:
pair = config_pair.split(':')
tmp_dict[pair[0]] = pair[1]
FLAGS.benchmark_config_pair = tmp_dict
try:
benchmark_list = benchmark_sets.GetBenchmarksFromFlags()
total_benchmarks = len(benchmark_list)
if FLAGS.parallelism > 1:
sequence_range = range(total_benchmarks, 0, -1)
args = [((benchmark, collector, sequence_counter,
total_benchmarks), {})
for benchmark, sequence_counter in zip(
benchmark_list, sequence_range)]
vm_util.RunThreaded(RunBenchmark,
args,
max_concurrent_threads=FLAGS.parallelism)
else:
sequence_range = range(1, total_benchmarks + 1)
for benchmark, sequence_counter in zip(benchmark_list,
sequence_range):
RunBenchmark(benchmark, collector, sequence_counter,
total_benchmarks)
finally:
if collector.samples:
collector.PublishSamples()
if FLAGS.run_stage not in [STAGE_ALL, STAGE_CLEANUP]:
logging.info('To run again with this setup, please use --run_uri=%s',
FLAGS.run_uri)