def RunBenchmark(spec, collector):
"""Runs a single benchmark and adds the results to the collector.
Args:
spec: The BenchmarkSpec object with run information.
collector: The SampleCollector object to add samples to.
"""
spec.status = benchmark_status.FAILED
# Modify the logger prompt for messages logged within this function.
label_extension = '{}({}/{})'.format(spec.name, spec.sequence_number,
spec.total_benchmarks)
context.SetThreadBenchmarkSpec(spec)
log_context = log_util.GetThreadLogContext()
with log_context.ExtendLabel(label_extension):
with spec.RedirectGlobalFlags():
end_to_end_timer = timing_util.IntervalTimer()
detailed_timer = timing_util.IntervalTimer()
try:
with end_to_end_timer.Measure('End to End'):
if stages.PROVISION in FLAGS.run_stage:
DoProvisionPhase(spec, detailed_timer)
if stages.PREPARE in FLAGS.run_stage:
DoPreparePhase(spec, detailed_timer)
if stages.RUN in FLAGS.run_stage:
DoRunPhase(spec, collector, detailed_timer)
if stages.CLEANUP in FLAGS.run_stage:
DoCleanupPhase(spec, detailed_timer)
if stages.TEARDOWN in FLAGS.run_stage:
DoTeardownPhase(spec, detailed_timer)
# Add timing samples.
if (FLAGS.run_stage == stages.STAGES
and timing_util.EndToEndRuntimeMeasurementEnabled()):
collector.AddSamples(end_to_end_timer.GenerateSamples(),
spec.name, spec)
if timing_util.RuntimeMeasurementsEnabled():
collector.AddSamples(detailed_timer.GenerateSamples(),
spec.name, spec)
except:
# Resource cleanup (below) can take a long time. Log the error to give
# immediate feedback, then re-throw.
logging.exception('Error during benchmark %s', spec.name)
# If the particular benchmark requests us to always call cleanup, do it
# here.
if stages.CLEANUP in FLAGS.run_stage and spec.always_call_cleanup:
DoCleanupPhase(spec, detailed_timer)
raise
finally:
if stages.TEARDOWN in FLAGS.run_stage:
spec.Delete()
events.benchmark_end.send(benchmark_spec=spec)
# Pickle spec to save final resource state.
spec.Pickle()
spec.status = benchmark_status.SUCCEEDED
def RunBenchmark(benchmark, collector, sequence_number, total_benchmarks):
"""Runs a single benchmark and adds the results to the collector.
Args:
benchmark: The benchmark module to be run.
collector: The SampleCollector object to add samples to.
sequence_number: The sequence number of when the benchmark was started
relative to the other benchmarks in the suite.
total_benchmarks: The total number of benchmarks in the suite.
"""
benchmark_info = benchmark.GetInfo()
if not ValidateBenchmarkInfo(benchmark_info):
return
benchmark_name = benchmark_info['name']
# Modify the logger prompt for messages logged within this function.
label_extension = '{}({}/{})'.format(benchmark_name, sequence_number,
total_benchmarks)
log_context = log_util.GetThreadLogContext()
with log_context.ExtendLabel(label_extension):
# Optional prerequisite checking.
check_prereqs = getattr(benchmark, 'CheckPrerequisites', None)
if check_prereqs:
try:
check_prereqs()
except:
logging.exception('Prerequisite check failed for %s',
benchmark_name)
raise
end_to_end_timer = timing_util.IntervalTimer()
detailed_timer = timing_util.IntervalTimer()
spec = None
try:
with end_to_end_timer.Measure('End to End'):
if FLAGS.run_stage in [STAGE_ALL, STAGE_PREPARE]:
# It is important to create the spec outside of DoPreparePhase
# because if DoPreparePhase raises an exception, we still need
# a reference to the spec in order to delete it in the "finally"
# section below.
spec = benchmark_spec.BenchmarkSpec(benchmark_info)
DoPreparePhase(benchmark, benchmark_name, spec,
detailed_timer)
else:
spec = benchmark_spec.BenchmarkSpec.GetSpecFromFile(
benchmark_name)
if FLAGS.run_stage in [STAGE_ALL, STAGE_RUN]:
DoRunPhase(benchmark, benchmark_name, spec, collector,
detailed_timer)
if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]:
DoCleanupPhase(benchmark, benchmark_name, spec,
detailed_timer)
# Add samples for any timed interval that was measured.
include_end_to_end = timing_util.EndToEndRuntimeMeasurementEnabled(
)
include_runtimes = timing_util.RuntimeMeasurementsEnabled()
include_timestamps = timing_util.TimestampMeasurementsEnabled()
if FLAGS.run_stage == STAGE_ALL:
collector.AddSamples(
end_to_end_timer.GenerateSamples(
include_runtime=include_end_to_end or include_runtimes,
include_timestamps=include_timestamps), benchmark_name,
spec)
collector.AddSamples(
detailed_timer.GenerateSamples(include_runtimes,
include_timestamps),
benchmark_name, spec)
except Exception:
# Resource cleanup (below) can take a long time. Log the error to give
# immediate feedback, then re-throw.
logging.exception('Error during benchmark %s', benchmark_name)
# If the particular benchmark requests us to always call cleanup, do it
# here.
if (FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP] and spec
and spec.always_call_cleanup):
DoCleanupPhase(benchmark, benchmark_name, spec, detailed_timer)
raise
finally:
if spec:
if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]:
spec.Delete()
# Pickle spec to save final resource state.
spec.PickleSpec()
def RunBenchmark(spec, collector):
"""Runs a single benchmark and adds the results to the collector.
Args:
spec: The BenchmarkSpec object with run information.
collector: The SampleCollector object to add samples to.
"""
# Since there are issues with the handling SIGINT/KeyboardInterrupt (see
# further dicussion in _BackgroundProcessTaskManager) this mechanism is
# provided for defense in depth to force skip pending runs after SIGINT.
for f in _SKIP_PENDING_RUNS_CHECKS:
if f():
logging.warning('Skipping benchmark.')
return
spec.status = benchmark_status.FAILED
current_run_stage = stages.PROVISION
# Modify the logger prompt for messages logged within this function.
label_extension = '{}({}/{})'.format(
spec.name, spec.sequence_number, spec.total_benchmarks)
context.SetThreadBenchmarkSpec(spec)
log_context = log_util.GetThreadLogContext()
with log_context.ExtendLabel(label_extension):
with spec.RedirectGlobalFlags():
end_to_end_timer = timing_util.IntervalTimer()
detailed_timer = timing_util.IntervalTimer()
try:
with end_to_end_timer.Measure('End to End'):
if stages.PROVISION in FLAGS.run_stage:
DoProvisionPhase(spec, detailed_timer)
if stages.PREPARE in FLAGS.run_stage:
current_run_stage = stages.PREPARE
DoPreparePhase(spec, detailed_timer)
if stages.RUN in FLAGS.run_stage:
current_run_stage = stages.RUN
DoRunPhase(spec, collector, detailed_timer)
if stages.CLEANUP in FLAGS.run_stage:
current_run_stage = stages.CLEANUP
DoCleanupPhase(spec, detailed_timer)
if stages.TEARDOWN in FLAGS.run_stage:
current_run_stage = stages.TEARDOWN
DoTeardownPhase(spec, detailed_timer)
# Add timing samples.
if (FLAGS.run_stage == stages.STAGES and
timing_util.EndToEndRuntimeMeasurementEnabled()):
collector.AddSamples(
end_to_end_timer.GenerateSamples(), spec.name, spec)
if timing_util.RuntimeMeasurementsEnabled():
collector.AddSamples(
detailed_timer.GenerateSamples(), spec.name, spec)
# Add resource related samples.
collector.AddSamples(spec.GetSamples(), spec.name, spec)
except Exception as e:
# Log specific type of failure, if known
# TODO(dlott) Move to exception chaining with Python3 support
if (isinstance(e, errors.Benchmarks.InsufficientCapacityCloudFailure)
or 'InsufficientCapacityCloudFailure' in str(e)):
spec.failed_substatus = (
benchmark_status.FailedSubstatus.INSUFFICIENT_CAPACITY)
spec.status_detail = str(e)
elif (isinstance(e, errors.Benchmarks.QuotaFailure)
or 'QuotaFailure' in str(e)):
spec.failed_substatus = benchmark_status.FailedSubstatus.QUOTA
spec.status_detail = str(e)
# Resource cleanup (below) can take a long time. Log the error to give
# immediate feedback, then re-throw.
logging.exception('Error during benchmark %s', spec.name)
if FLAGS.create_failed_run_samples:
collector.AddSamples(MakeFailedRunSample(spec, str(e),
current_run_stage),
spec.name,
spec)
# If the particular benchmark requests us to always call cleanup, do it
# here.
if stages.CLEANUP in FLAGS.run_stage and spec.always_call_cleanup:
DoCleanupPhase(spec, detailed_timer)
raise
finally:
# Deleting resources should happen first so any errors with publishing
# don't prevent teardown.
if stages.TEARDOWN in FLAGS.run_stage:
spec.Delete()
if FLAGS.publish_after_run:
collector.PublishSamples()
events.benchmark_end.send(benchmark_spec=spec)
# Pickle spec to save final resource state.
spec.Pickle()
spec.status = benchmark_status.SUCCEEDED
def RunBenchmark(benchmark, sequence_number, total_benchmarks,
benchmark_config, benchmark_uid, collector):
"""Runs a single benchmark and adds the results to the collector.
Args:
benchmark: The benchmark module to be run.
sequence_number: The sequence number of when the benchmark was started
relative to the other benchmarks in the suite.
total_benchmarks: The total number of benchmarks in the suite.
benchmark_config: BenchmarkConfigSpec. The config to run the benchmark with.
benchmark_uid: An identifier unique to this run of the benchmark even
if the same benchmark is run multiple times with different configs.
collector: The SampleCollector object to add samples to.
"""
benchmark_name = benchmark.BENCHMARK_NAME
# Modify the logger prompt for messages logged within this function.
label_extension = '{}({}/{})'.format(benchmark_name, sequence_number,
total_benchmarks)
log_context = log_util.GetThreadLogContext()
with log_context.ExtendLabel(label_extension):
spec = _GetBenchmarkSpec(benchmark_config, benchmark_name,
benchmark_uid)
with spec.RedirectGlobalFlags():
end_to_end_timer = timing_util.IntervalTimer()
detailed_timer = timing_util.IntervalTimer()
try:
with end_to_end_timer.Measure('End to End'):
if stages.PROVISION in FLAGS.run_stage:
DoProvisionPhase(benchmark_name, spec, detailed_timer)
if stages.PREPARE in FLAGS.run_stage:
DoPreparePhase(benchmark, benchmark_name, spec,
detailed_timer)
if stages.RUN in FLAGS.run_stage:
DoRunPhase(benchmark, benchmark_name, spec, collector,
detailed_timer)
if stages.CLEANUP in FLAGS.run_stage:
DoCleanupPhase(benchmark, benchmark_name, spec,
detailed_timer)
if stages.TEARDOWN in FLAGS.run_stage:
DoTeardownPhase(benchmark_name, spec, detailed_timer)
# Add samples for any timed interval that was measured.
include_end_to_end = timing_util.EndToEndRuntimeMeasurementEnabled(
)
include_runtimes = timing_util.RuntimeMeasurementsEnabled()
include_timestamps = timing_util.TimestampMeasurementsEnabled()
if FLAGS.run_stage == stages.STAGES:
# Ran all stages.
collector.AddSamples(
end_to_end_timer.GenerateSamples(
include_runtime=include_end_to_end
or include_runtimes,
include_timestamps=include_timestamps),
benchmark_name, spec)
collector.AddSamples(
detailed_timer.GenerateSamples(include_runtimes,
include_timestamps),
benchmark_name, spec)
except:
# Resource cleanup (below) can take a long time. Log the error to give
# immediate feedback, then re-throw.
logging.exception('Error during benchmark %s', benchmark_name)
# If the particular benchmark requests us to always call cleanup, do it
# here.
if stages.CLEANUP in FLAGS.run_stage and spec.always_call_cleanup:
DoCleanupPhase(benchmark, benchmark_name, spec,
detailed_timer)
raise
finally:
if stages.TEARDOWN in FLAGS.run_stage:
spec.Delete()
events.benchmark_end.send(benchmark_spec=spec)
# Pickle spec to save final resource state.
spec.PickleSpec()
def RunBenchmark(benchmark, collector, sequence_number, total_benchmarks,
benchmark_config, benchmark_uid):
"""Runs a single benchmark and adds the results to the collector.
Args:
benchmark: The benchmark module to be run.
collector: The SampleCollector object to add samples to.
sequence_number: The sequence number of when the benchmark was started
relative to the other benchmarks in the suite.
total_benchmarks: The total number of benchmarks in the suite.
benchmark_config: The config to run the benchmark with.
benchmark_uid: An identifier unique to this run of the benchmark even
if the same benchmark is run multiple times with different configs.
"""
benchmark_name = benchmark.BENCHMARK_NAME
# Modify the logger prompt for messages logged within this function.
label_extension = '{}({}/{})'.format(benchmark_name, sequence_number,
total_benchmarks)
log_context = log_util.GetThreadLogContext()
with log_context.ExtendLabel(label_extension):
# Optional prerequisite checking.
check_prereqs = getattr(benchmark, 'CheckPrerequisites', None)
if check_prereqs:
try:
check_prereqs()
except:
logging.exception('Prerequisite check failed for %s',
benchmark_name)
raise
end_to_end_timer = timing_util.IntervalTimer()
detailed_timer = timing_util.IntervalTimer()
spec = None
try:
with end_to_end_timer.Measure('End to End'):
if FLAGS.run_stage in [STAGE_ALL, STAGE_PROVISION]:
# It is important to create the spec outside of DoProvisionPhase
# because if DoPreparePhase raises an exception, we still need
# a reference to the spec in order to delete it in the "finally"
# section below.
spec = benchmark_spec.BenchmarkSpec(
benchmark_config, benchmark_name, benchmark_uid)
spec.ConstructVirtualMachines()
DoProvisionPhase(benchmark_name, spec, detailed_timer)
else:
try:
spec = benchmark_spec.BenchmarkSpec.GetSpecFromFile(
benchmark_uid)
except IOError:
if FLAGS.run_stage == STAGE_PREPARE:
logging.error(
'We were unable to load the BenchmarkSpec. This may be '
'related to two additional run stages which have recently '
'been added. Please make sure to run the stage "provision" '
'before "prepare". Similarly, make sure to run "teardown" '
'after "cleanup".')
raise
if FLAGS.run_stage in [STAGE_ALL, STAGE_PREPARE]:
DoPreparePhase(benchmark, benchmark_name, spec,
detailed_timer)
if FLAGS.run_stage in [STAGE_ALL, STAGE_RUN]:
DoRunPhase(benchmark, benchmark_name, spec, collector,
detailed_timer)
if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]:
DoCleanupPhase(benchmark, benchmark_name, spec,
detailed_timer)
if FLAGS.run_stage in [STAGE_ALL, STAGE_TEARDOWN]:
DoTeardownPhase(benchmark_name, spec, detailed_timer)
# Add samples for any timed interval that was measured.
include_end_to_end = timing_util.EndToEndRuntimeMeasurementEnabled(
)
include_runtimes = timing_util.RuntimeMeasurementsEnabled()
include_timestamps = timing_util.TimestampMeasurementsEnabled()
if FLAGS.run_stage == STAGE_ALL:
collector.AddSamples(
end_to_end_timer.GenerateSamples(
include_runtime=include_end_to_end or include_runtimes,
include_timestamps=include_timestamps), benchmark_name,
spec)
collector.AddSamples(
detailed_timer.GenerateSamples(include_runtimes,
include_timestamps),
benchmark_name, spec)
except:
# Resource cleanup (below) can take a long time. Log the error to give
# immediate feedback, then re-throw.
logging.exception('Error during benchmark %s', benchmark_name)
# If the particular benchmark requests us to always call cleanup, do it
# here.
if (FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP] and spec
and spec.always_call_cleanup):
DoCleanupPhase(benchmark, benchmark_name, spec, detailed_timer)
raise
finally:
if spec:
if FLAGS.run_stage in [STAGE_ALL, STAGE_TEARDOWN]:
spec.Delete()
# Pickle spec to save final resource state.
spec.PickleSpec()
