def RunBenchmark(benchmark, sequence_number, total_benchmarks, benchmark_config, benchmark_uid, collector): """Runs a single benchmark and adds the results to the collector. Args: benchmark: The benchmark module to be run. sequence_number: The sequence number of when the benchmark was started relative to the other benchmarks in the suite. total_benchmarks: The total number of benchmarks in the suite. benchmark_config: BenchmarkConfigSpec. The config to run the benchmark with. benchmark_uid: An identifier unique to this run of the benchmark even if the same benchmark is run multiple times with different configs. collector: The SampleCollector object to add samples to. """ benchmark_name = benchmark.BENCHMARK_NAME # Modify the logger prompt for messages logged within this function. label_extension = '{}({}/{})'.format(benchmark_name, sequence_number, total_benchmarks) log_context = log_util.GetThreadLogContext() with log_context.ExtendLabel(label_extension): spec = _GetBenchmarkSpec(benchmark_config, benchmark_name, benchmark_uid) with spec.RedirectGlobalFlags(): end_to_end_timer = timing_util.IntervalTimer() detailed_timer = timing_util.IntervalTimer() try: with end_to_end_timer.Measure('End to End'): if stages.PROVISION in FLAGS.run_stage: DoProvisionPhase(benchmark_name, spec, detailed_timer) if stages.PREPARE in FLAGS.run_stage: DoPreparePhase(benchmark, benchmark_name, spec, detailed_timer) if stages.RUN in FLAGS.run_stage: DoRunPhase(benchmark, benchmark_name, spec, collector, detailed_timer) if stages.CLEANUP in FLAGS.run_stage: DoCleanupPhase(benchmark, benchmark_name, spec, detailed_timer) if stages.TEARDOWN in FLAGS.run_stage: DoTeardownPhase(benchmark_name, spec, detailed_timer) # Add samples for any timed interval that was measured. include_end_to_end = timing_util.EndToEndRuntimeMeasurementEnabled( ) include_runtimes = timing_util.RuntimeMeasurementsEnabled() include_timestamps = timing_util.TimestampMeasurementsEnabled() if FLAGS.run_stage == stages.STAGES: # Ran all stages. collector.AddSamples( end_to_end_timer.GenerateSamples( include_runtime=include_end_to_end or include_runtimes, include_timestamps=include_timestamps), benchmark_name, spec) collector.AddSamples( detailed_timer.GenerateSamples(include_runtimes, include_timestamps), benchmark_name, spec) except: # Resource cleanup (below) can take a long time. Log the error to give # immediate feedback, then re-throw. logging.exception('Error during benchmark %s', benchmark_name) # If the particular benchmark requests us to always call cleanup, do it # here. if stages.CLEANUP in FLAGS.run_stage and spec.always_call_cleanup: DoCleanupPhase(benchmark, benchmark_name, spec, detailed_timer) raise finally: if stages.TEARDOWN in FLAGS.run_stage: spec.Delete() events.benchmark_end.send(benchmark_spec=spec) # Pickle spec to save final resource state. spec.PickleSpec()
def RunBenchmark(benchmark, collector, sequence_number, total_benchmarks): """Runs a single benchmark and adds the results to the collector. Args: benchmark: The benchmark module to be run. collector: The SampleCollector object to add samples to. sequence_number: The sequence number of when the benchmark was started relative to the other benchmarks in the suite. total_benchmarks: The total number of benchmarks in the suite. """ benchmark_info = benchmark.GetInfo() if not ValidateBenchmarkInfo(benchmark_info): return benchmark_name = benchmark_info['name'] # Modify the logger prompt for messages logged within this function. label_extension = '{}({}/{})'.format(benchmark_name, sequence_number, total_benchmarks) log_context = log_util.GetThreadLogContext() with log_context.ExtendLabel(label_extension): # Optional prerequisite checking. check_prereqs = getattr(benchmark, 'CheckPrerequisites', None) if check_prereqs: try: check_prereqs() except: logging.exception('Prerequisite check failed for %s', benchmark_name) raise end_to_end_timer = timing_util.IntervalTimer() detailed_timer = timing_util.IntervalTimer() spec = None try: with end_to_end_timer.Measure('End to End'): if FLAGS.run_stage in [STAGE_ALL, STAGE_PREPARE]: # It is important to create the spec outside of DoPreparePhase # because if DoPreparePhase raises an exception, we still need # a reference to the spec in order to delete it in the "finally" # section below. spec = benchmark_spec.BenchmarkSpec(benchmark_info) DoPreparePhase(benchmark, benchmark_name, spec, detailed_timer) else: spec = benchmark_spec.BenchmarkSpec.GetSpecFromFile( benchmark_name) if FLAGS.run_stage in [STAGE_ALL, STAGE_RUN]: DoRunPhase(benchmark, benchmark_name, spec, collector, detailed_timer) if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]: DoCleanupPhase(benchmark, benchmark_name, spec, detailed_timer) # Add samples for any timed interval that was measured. include_end_to_end = timing_util.EndToEndRuntimeMeasurementEnabled( ) include_runtimes = timing_util.RuntimeMeasurementsEnabled() include_timestamps = timing_util.TimestampMeasurementsEnabled() if FLAGS.run_stage == STAGE_ALL: collector.AddSamples( end_to_end_timer.GenerateSamples( include_runtime=include_end_to_end or include_runtimes, include_timestamps=include_timestamps), benchmark_name, spec) collector.AddSamples( detailed_timer.GenerateSamples(include_runtimes, include_timestamps), benchmark_name, spec) except Exception: # Resource cleanup (below) can take a long time. Log the error to give # immediate feedback, then re-throw. logging.exception('Error during benchmark %s', benchmark_name) # If the particular benchmark requests us to always call cleanup, do it # here. if (FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP] and spec and spec.always_call_cleanup): DoCleanupPhase(benchmark, benchmark_name, spec, detailed_timer) raise finally: if spec: if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]: spec.Delete() # Pickle spec to save final resource state. spec.PickleSpec()
def RunBenchmark(benchmark, collector, sequence_number, total_benchmarks, benchmark_config, benchmark_uid): """Runs a single benchmark and adds the results to the collector. Args: benchmark: The benchmark module to be run. collector: The SampleCollector object to add samples to. sequence_number: The sequence number of when the benchmark was started relative to the other benchmarks in the suite. total_benchmarks: The total number of benchmarks in the suite. benchmark_config: The config to run the benchmark with. benchmark_uid: An identifier unique to this run of the benchmark even if the same benchmark is run multiple times with different configs. """ benchmark_name = benchmark.BENCHMARK_NAME # Modify the logger prompt for messages logged within this function. label_extension = '{}({}/{})'.format(benchmark_name, sequence_number, total_benchmarks) log_context = log_util.GetThreadLogContext() with log_context.ExtendLabel(label_extension): # Optional prerequisite checking. check_prereqs = getattr(benchmark, 'CheckPrerequisites', None) if check_prereqs: try: check_prereqs() except: logging.exception('Prerequisite check failed for %s', benchmark_name) raise end_to_end_timer = timing_util.IntervalTimer() detailed_timer = timing_util.IntervalTimer() spec = None try: with end_to_end_timer.Measure('End to End'): if FLAGS.run_stage in [STAGE_ALL, STAGE_PROVISION]: # It is important to create the spec outside of DoProvisionPhase # because if DoPreparePhase raises an exception, we still need # a reference to the spec in order to delete it in the "finally" # section below. spec = benchmark_spec.BenchmarkSpec( benchmark_config, benchmark_name, benchmark_uid) spec.ConstructVirtualMachines() DoProvisionPhase(benchmark_name, spec, detailed_timer) else: try: spec = benchmark_spec.BenchmarkSpec.GetSpecFromFile( benchmark_uid) except IOError: if FLAGS.run_stage == STAGE_PREPARE: logging.error( 'We were unable to load the BenchmarkSpec. This may be ' 'related to two additional run stages which have recently ' 'been added. Please make sure to run the stage "provision" ' 'before "prepare". Similarly, make sure to run "teardown" ' 'after "cleanup".') raise if FLAGS.run_stage in [STAGE_ALL, STAGE_PREPARE]: DoPreparePhase(benchmark, benchmark_name, spec, detailed_timer) if FLAGS.run_stage in [STAGE_ALL, STAGE_RUN]: DoRunPhase(benchmark, benchmark_name, spec, collector, detailed_timer) if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]: DoCleanupPhase(benchmark, benchmark_name, spec, detailed_timer) if FLAGS.run_stage in [STAGE_ALL, STAGE_TEARDOWN]: DoTeardownPhase(benchmark_name, spec, detailed_timer) # Add samples for any timed interval that was measured. include_end_to_end = timing_util.EndToEndRuntimeMeasurementEnabled( ) include_runtimes = timing_util.RuntimeMeasurementsEnabled() include_timestamps = timing_util.TimestampMeasurementsEnabled() if FLAGS.run_stage == STAGE_ALL: collector.AddSamples( end_to_end_timer.GenerateSamples( include_runtime=include_end_to_end or include_runtimes, include_timestamps=include_timestamps), benchmark_name, spec) collector.AddSamples( detailed_timer.GenerateSamples(include_runtimes, include_timestamps), benchmark_name, spec) except: # Resource cleanup (below) can take a long time. Log the error to give # immediate feedback, then re-throw. logging.exception('Error during benchmark %s', benchmark_name) # If the particular benchmark requests us to always call cleanup, do it # here. if (FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP] and spec and spec.always_call_cleanup): DoCleanupPhase(benchmark, benchmark_name, spec, detailed_timer) raise finally: if spec: if FLAGS.run_stage in [STAGE_ALL, STAGE_TEARDOWN]: spec.Delete() # Pickle spec to save final resource state. spec.PickleSpec()