def testFlagOverride(self):
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
NAME,
flag_values=FLAGS,
flags={'benchmark_spec_test_flag': 1},
vm_groups={})
spec = benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec, UID)
self.assertEqual(FLAGS.benchmark_spec_test_flag, 0)
with spec.RedirectGlobalFlags():
self.assertEqual(FLAGS.benchmark_spec_test_flag, 1)
FLAGS.benchmark_spec_test_flag = 2
self.assertEqual(FLAGS.benchmark_spec_test_flag, 2)
self.assertEqual(FLAGS.benchmark_spec_test_flag, 0)
def testBackgroundWorkloadVanillaConfig(self):
""" Test that nothing happens with the vanilla config """
with mock_flags.PatchFlags() as mocked_flags:
self.setupCommonFlags(mocked_flags)
mocked_flags.background_cpu_threads = None
config = configs.LoadConfig(ping_benchmark.BENCHMARK_CONFIG, {},
NAME)
spec = benchmark_spec.BenchmarkSpec(config, NAME, UID)
spec.ConstructVirtualMachines()
for vm in spec.vms:
self.assertIsNone(vm.background_cpu_threads)
self._CheckVMFromSpec(spec, 0)
def setUp(self):
super(_DiskMetadataTestCase, self).setUp()
self.addCleanup(context.SetThreadBenchmarkSpec, None)
p = mock.patch(vm_util.__name__ + '.GetTempDir',
return_value='/tmp/dir')
p.start()
self.addCleanup(p.stop)
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
_BENCHMARK_NAME, flag_values=FLAGS, vm_groups={})
self.benchmark_spec = benchmark_spec.BenchmarkSpec(
mock.MagicMock(), config_spec, _BENCHMARK_UID)
def setUp(self):
self._mocked_flags = mock_flags.PatchTestCaseFlags(self)
self._mocked_flags.cloud = providers.GCP
self._mocked_flags.gcloud_path = 'test_gcloud'
self._mocked_flags.os_type = os_types.DEBIAN
self._mocked_flags.run_uri = 'aaaaaa'
self._mocked_flags.gcp_instance_metadata = []
self._mocked_flags.gcp_instance_metadata_from_file = []
# Creating a VM object causes network objects to be added to the current
# thread's benchmark spec. Create such a benchmark spec for these tests.
self.addCleanup(context.SetThreadBenchmarkSpec, None)
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
_BENCHMARK_NAME, flag_values=self._mocked_flags, vm_groups={})
self._benchmark_spec = benchmark_spec.BenchmarkSpec(
config_spec, _BENCHMARK_NAME, _BENCHMARK_UID)
def testStaticVms(self):
config = configs.LoadConfig(STATIC_VM_CONFIG, {}, NAME)
spec = benchmark_spec.BenchmarkSpec(config, NAME, UID)
spec.ConstructVirtualMachines()
self.assertEqual(len(spec.vms), 4)
vm0 = spec.vm_groups['group1'][0]
vm1, vm2, vm3 = spec.vm_groups['group2']
self.assertIsInstance(vm0, gce_vm.GceVirtualMachine)
self.assertIsInstance(vm1, static_vm.StaticVirtualMachine)
self.assertIsInstance(vm2, static_vm.RhelBasedStaticVirtualMachine)
self.assertIsInstance(vm3, gce_vm.GceVirtualMachine)
self.assertEqual(vm2.disk_specs[0].mount_point, '/scratch')
def testWindowsVMCausesError(self):
""" windows vm with background_cpu_threads raises exception """
with mock_flags.PatchFlags() as mocked_flags:
self.setupCommonFlags(mocked_flags)
mocked_flags.background_cpu_threads = 1
mocked_flags.os_type = benchmark_spec.WINDOWS
config = configs.LoadConfig(ping_benchmark.BENCHMARK_CONFIG, {},
NAME)
spec = benchmark_spec.BenchmarkSpec(config, NAME, UID)
spec.ConstructVirtualMachines()
with self.assertRaises(Exception):
spec.Prepare()
with self.assertRaises(Exception):
spec.StartBackgroundWorkload()
with self.assertRaises(Exception):
spec.StopBackgroundWorkload()
def testBenchmarkSpecLoadsProvider(self):
p = mock.patch(providers.__name__ + '.LoadProvider')
p.start()
self.addCleanup(p.stop)
config = {
'vm_groups': {
'group1': {
'cloud': 'AWS',
'vm_count': 0,
'vm_spec': {'AWS': {}}
}
}
}
spec = benchmark_spec.BenchmarkSpec(config, 'name', 'uid')
spec.ConstructVirtualMachines()
providers.LoadProvider.assert_called_with('aws')
def assertDiskMounts(benchmark_config, mount_point):
"""Test whether a disk mounts in a given configuration.
Sets up a virtual machine following benchmark_config and then tests
whether the path mount_point contains a working disk by trying to
create a file there. Returns nothing if file creation works;
otherwise raises an exception.
Args:
benchmark_config: a dict in the format of
benchmark_spec.BenchmarkSpec. The config must specify exactly
one virtual machine.
mount_point: a path, represented as a string.
Raises:
RemoteCommandError if it cannot create a file at mount_point and
verify that the file exists.
AssertionError if benchmark_config does not specify exactly one
virtual machine.
"""
assert len(benchmark_config['vm_groups']) == 1
vm_group = benchmark_config['vm_groups'].itervalues().next()
assert vm_group.get('num_vms', 1) == 1
m = mock.MagicMock()
m.BENCHMARK_NAME = _BENCHMARK_NAME
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
_BENCHMARK_NAME, flag_values=flags.FLAGS, **benchmark_config)
spec = benchmark_spec.BenchmarkSpec(
m, config_spec, _BENCHMARK_UID)
with spec.RedirectGlobalFlags():
try:
spec.ConstructVirtualMachines()
spec.Provision()
vm = spec.vms[0]
test_file_path = os.path.join(mount_point, 'test_file')
vm.RemoteCommand('touch %s' % test_file_path)
# This will raise RemoteCommandError if the test file does not
# exist.
vm.RemoteCommand('test -e %s' % test_file_path)
finally:
spec.Delete()
def setUp(self):
super(AwsVirtualMachineTestCase, self).setUp()
FLAGS.cloud = providers.AWS
FLAGS.os_type = os_types.DEBIAN
FLAGS.run_uri = 'aaaaaa'
FLAGS.temp_dir = 'tmp'
p = mock.patch('perfkitbenchmarker.providers.aws.'
'util.IssueRetryableCommand')
p.start()
self.addCleanup(p.stop)
p2 = mock.patch('perfkitbenchmarker.' 'vm_util.IssueCommand')
p2.start()
self.addCleanup(p2.stop)
# VM Creation depends on there being a BenchmarkSpec.
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
_BENCHMARK_NAME, flag_values=FLAGS, vm_groups={})
self.spec = benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec,
_BENCHMARK_UID)
self.addCleanup(context.SetThreadBenchmarkSpec, None)
self.vm = aws_virtual_machine.AwsVirtualMachine(
aws_virtual_machine.AwsVmSpec('test_vm_spec.AWS',
zone='us-east-1a',
machine_type='c3.large',
spot_price=123.45))
self.vm.id = 'i-foo'
self.vm.image = 'ami-12345'
self.vm.client_token = '00000000-1111-2222-3333-444444444444'
network_mock = mock.MagicMock()
network_mock.subnet = mock.MagicMock(id='subnet-id')
placement_group = mock.MagicMock()
placement_group.name = 'placement_group_name'
placement_group.strategy = 'cluster'
network_mock.placement_group = placement_group
self.vm.network = network_mock
self.vm.placement_group = placement_group
self.response = self.open_json_data('aws-describe-instance.json')
self.sir_response = self.open_json_data(
'aws-describe-spot-instance-requests.json')
self.vm.network.is_static = False
self.vm.network.regional_network.vpc.default_security_group_id = 'sg-1234'
def setUp(self):
super(GCEVMFlagsTestCase, self).setUp()
FLAGS.cloud = providers.GCP
FLAGS.gcloud_path = 'test_gcloud'
FLAGS.run_uri = 'aaaaaa'
FLAGS.gcp_instance_metadata = []
FLAGS.gcp_instance_metadata_from_file = []
# Creating a VM object causes network objects to be added to the current
# thread's benchmark spec. Create such a benchmark spec for these tests.
self.addCleanup(context.SetThreadBenchmarkSpec, None)
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
_BENCHMARK_NAME, flag_values=FLAGS, vm_groups={})
self._benchmark_spec = benchmark_spec.BenchmarkSpec(
mock.MagicMock(), config_spec, _BENCHMARK_UID)
get_tmp_dir_mock = mock.patch(vm_util.__name__ + '.GetTempDir',
return_value='TempDir')
get_tmp_dir_mock.start()
self.addCleanup(get_tmp_dir_mock.stop)
def DoPreparePhase(benchmark, info, name, timer):
"""Performs the Prepare phase of benchmark execution.
Args:
benchmark: The benchmark module.
info: The dict returned by the benchmark module's GetInfo function.
name: A string containing the benchmark name.
timer: An IntervalTimer that measures the start and stop times of resource
provisioning and the benchmark module's Prepare function.
Returns:
The BenchmarkSpec created for the benchmark.
"""
logging.info('Preparing benchmark %s', name)
with timer.Measure('Resource Provisioning'):
spec = benchmark_spec.BenchmarkSpec(info)
spec.Prepare()
with timer.Measure('Benchmark Prepare'):
benchmark.Prepare(spec)
return spec
def doAwsDiskTest(self, disk_type, machine_type, goal_media,
goal_replication, goal_legacy_disk_type):
disk_spec = aws_disk.AwsDiskSpec(_COMPONENT,
disk_size=2,
disk_type=disk_type)
context.SetThreadBenchmarkSpec(
benchmark_spec.BenchmarkSpec({}, 'name', 'uid'))
vm_spec = virtual_machine.BaseVmSpec('test_vm_spec.AWS',
zone='us-east-1a',
machine_type=machine_type)
vm = aws_virtual_machine.DebianBasedAwsVirtualMachine(vm_spec)
vm.CreateScratchDisk(disk_spec)
self.assertEqual(
vm.scratch_disks[0].metadata, {
disk.MEDIA: goal_media,
disk.REPLICATION: goal_replication,
disk.LEGACY_DISK_TYPE: goal_legacy_disk_type
})
def setUp(self):
mocked_flags = mock_flags.PatchTestCaseFlags(self)
mocked_flags.cloud = providers.AWS
mocked_flags.os_type = os_types.DEBIAN
mocked_flags.run_uri = 'aaaaaa'
mocked_flags.temp_dir = 'tmp'
p = mock.patch('perfkitbenchmarker.providers.aws.'
'util.IssueRetryableCommand')
p.start()
self.addCleanup(p.stop)
p2 = mock.patch('perfkitbenchmarker.' 'vm_util.IssueCommand')
p2.start()
self.addCleanup(p2.stop)
# VM Creation depends on there being a BenchmarkSpec.
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
_BENCHMARK_NAME, flag_values=mocked_flags, vm_groups={})
self.spec = benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec,
_BENCHMARK_UID)
self.addCleanup(context.SetThreadBenchmarkSpec, None)
self.vm = aws_virtual_machine.AwsVirtualMachine(
aws_virtual_machine.AwsVmSpec('test_vm_spec.AWS',
zone='us-east-1a',
machine_type='c3.large',
spot_price=123.45))
self.vm.id = 'i-foo'
self.vm.image = 'ami-12345'
self.vm.client_token = '00000000-1111-2222-3333-444444444444'
network_mock = mock.MagicMock()
network_mock.subnet = mock.MagicMock(id='subnet-id')
placement_group = mock.MagicMock()
placement_group.name = 'placement_group_name'
network_mock.placement_group = placement_group
self.vm.network = network_mock
self.response = self.openJsonData('aws-describe-instance.json')
self.sir_response =\
self.openJsonData('aws-describe-spot-instance-requests.json')
def setUp(self):
self.saved_flag_values = flagsaver.save_flag_values()
self.patches = []
vm_prefix = linux_virtual_machine.__name__ + '.BaseLinuxMixin'
self.patches.append(
mock.patch(vm_prefix + '.FormatDisk'))
self.patches.append(
mock.patch(vm_prefix + '.MountDisk'))
self.patches.append(
mock.patch(
util.__name__ + '.GetDefaultProject', side_effect='test_project'))
# Patch subprocess.Popen to make sure we don't issue any commands to spin up
# resources.
self.patches.append(mock.patch('subprocess.Popen'))
self.patches.append(
mock.patch(vm_util.__name__ + '.GetTempDir', return_value='/tmp/dir'))
self._PatchCloudSpecific()
for p in self.patches:
p.start()
self.addCleanup(p.stop)
# We need the disk class mocks to return new mocks each time they are
# called. Otherwise all "disks" instantiated will be the same object.
self._GetDiskClass().side_effect = (
lambda *args, **kwargs: mock.MagicMock(is_striped=False))
# VM Creation depends on there being a BenchmarkSpec.
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
_BENCHMARK_NAME, flag_values=FLAGS, vm_groups={})
self.spec = benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec,
_BENCHMARK_UID)
self.addCleanup(context.SetThreadBenchmarkSpec, None)
self.addCleanup(flagsaver.restore_flag_values, self.saved_flag_values)
def _GetBenchmarkSpec(benchmark_config, benchmark_name, benchmark_uid):
"""Creates a BenchmarkSpec or loads one from a file.
During the provision stage, creates a BenchmarkSpec from the provided
configuration. During any later stage, loads the BenchmarkSpec that was
created during the provision stage from a file.
Args:
benchmark_config: A Python dictionary representation of the configuration
for the benchmark. For a complete explanation, see
perfkitbenchmarker/configs/__init__.py.
benchmark_name: string. Name of the benchmark.
benchmark_uid: string. Identifies a specific run of a benchmark.
Returns:
The created or loaded BenchmarkSpec.
"""
if FLAGS.run_stage in (STAGE_ALL, STAGE_PROVISION):
return benchmark_spec.BenchmarkSpec(benchmark_config, benchmark_name,
benchmark_uid)
else:
# TODO(skschneider): Build BenchmarkConfigSpec before RunBenchmark.
config = benchmark_config_spec.BenchmarkConfigSpec(benchmark_name,
flag_values=FLAGS,
**benchmark_config)
try:
return benchmark_spec.BenchmarkSpec.GetSpecFromFile(
benchmark_uid, config)
except IOError:
if FLAGS.run_stage == STAGE_PREPARE:
logging.error(
'We were unable to load the BenchmarkSpec. This may be related '
'to two additional run stages which have recently been added. '
'Please make sure to run the stage "provision" before "prepare". '
'Similarly, make sure to run "teardown" after "cleanup".')
raise
def testPDSSD(self):
config = {
'vm_groups': {
'vm_group_1': {
'cloud': 'GCP',
'vm_spec': {
'GCP': {
'machine_type': 'test_machine_type',
}
},
'disk_spec': {
'GCP': {
'disk_type': 'remote_ssd',
'disk_size': 2,
}
}
}
}
}
spec = benchmark_spec.BenchmarkSpec(config, 'name', 'uid')
spec.ConstructVirtualMachines()
self.assertEquals(spec.vms[0].disk_specs[0].disk_type, 'pd-ssd')
def testEBSStandard(self):
config = {
'vm_groups': {
'vm_group_1': {
'cloud': 'AWS',
'vm_spec': {
'AWS': {
'zone': 'us-east-1a'
}
},
'disk_spec': {
'AWS': {
'disk_type': 'standard',
'disk_size': 2
}
}
}
}
}
spec = benchmark_spec.BenchmarkSpec(config, 'name', 'uid')
spec.ConstructVirtualMachines()
self.assertEquals(spec.vms[0].disk_specs[0].disk_type, 'standard')
def testRun(self):
FLAGS.run_uri = '12345678'
FLAGS.zone = ['us-central1-a']
remote_command = mock.PropertyMock(return_value=(self.iperf_csv_text,
''))
client = mock.Mock(RemoteCommand=remote_command,
machine_type='n2-standard-2')
server = mock.Mock(internal_ip='10.0.0.1',
machine_type='n2-standard-4')
benchmark_module = mock.Mock(BENCHMARK_NAME='cloud_harmony_iperf')
benchmark_config = mock.Mock(vm_groups={},
relational_db=mock.Mock(vm_groups={}))
spec = benchmark_spec.BenchmarkSpec(benchmark_module, benchmark_config,
'abcdefg')
spec.vm_groups = {'client': [client], 'server': [server]}
results = cloudharmony_iperf_benchmark._Run(spec)
client.RobustRemoteCommand.assert_called_with(
'iperf/run.sh '
f'--iperf_server 10.0.0.1:{cloudharmony_iperf_benchmark._PORT} '
'--meta_compute_service Google Compute Engine '
'--meta_compute_service_id google:compute '
'--meta_instance_id n2-standard-2 '
'--meta_provider Google Cloud Platform '
'--meta_provider_id google '
'--meta_region us-central1 '
'--meta_zone us-central1-a '
'--meta_test_id 12345678 '
'--iperf_server_instance_id n2-standard-4 '
'--iperf_server_region us-central1-a '
'--iperf_time 10 '
'--iperf_warmup 0 '
'--iperf_test TCP '
'--iperf_parallel 1 '
'--tcp_bw_file /tmp/tcpbw '
'--wkhtml_xvfb '
'--verbose')
self.assertLen(results, 2)
expected_gartner_metadata = [{
'bandwidth_direction': 'up',
'bandwidth_max': 9894.384,
'bandwidth_mean': 9735.5,
'bandwidth_median': 9888.76,
'bandwidth_min': 8349.13,
'bandwidth_p10': 8349.13,
'bandwidth_p25': 9888.001,
'bandwidth_p75': 9889.2,
'bandwidth_p90': 9889.8,
'bandwidth_stdev': 487,
'benchmark_version': 1.0,
'collectd_rrd': '',
'cpu_client': 34.346987,
'cpu_server': 2.880331,
'iperf_bandwidth': '',
'iperf_cmd': 'iperf3 -c 10.240.0.14 -J -i 1',
'iperf_concurrency': 1,
'iperf_interval': 1,
'iperf_len': '',
'iperf_mss': '',
'iperf_nodelay': '',
'iperf_num': '',
'iperf_parallel': 1,
'iperf_reverse': '',
'iperf_server': '10.240.0.14',
'iperf_server_instance_id': 'n1-standard-2',
'iperf_server_os': 'Ubuntu 18.04.5 LTS',
'iperf_server_provider': '',
'iperf_server_provider_id': 'google',
'iperf_server_region': 'us-central1',
'iperf_server_service': '',
'iperf_server_service_id': 'google:compute',
'iperf_time': 10,
'iperf_tos': '',
'iperf_udp': '',
'iperf_version': '3.1.3',
'iperf_warmup': 0,
'iperf_window': '',
'iperf_zerocopy': '',
'iteration': 1,
'jitter_max': '',
'jitter_mean': '',
'jitter_median': '',
'jitter_min': '',
'jitter_p10': '',
'jitter_p25': '',
'jitter_p75': '',
'jitter_p90': '',
'jitter_stdev': '',
'loss_max': '',
'loss_mean': '',
'loss_median': '',
'loss_min': '',
'loss_p10': '',
'loss_p25': '',
'loss_p75': '',
'loss_p90': '',
'loss_stdev': '',
'meta_compute_service': '',
'meta_compute_service_id': 'google:compute',
'meta_cpu': 'Intel Xeon 2.30GHz',
'meta_cpu_cache': '46080 KB',
'meta_cpu_cores': 2,
'meta_cpu_speed': 2300.0,
'meta_hostname': 'pkb-3b246db4-0',
'meta_instance_id': 'n1-standard-2',
'meta_memory': '7 GB',
'meta_memory_gb': 7,
'meta_memory_mb': 7457,
'meta_os': 'Ubuntu 18.04.5 LTS',
'meta_provider': '',
'meta_provider_id': 'google',
'meta_region': 'us-central1',
'meta_resource_id': '',
'meta_run_group_id': '',
'meta_run_id': '',
'meta_test_id': '3b246db4',
'report_pdf': '',
'report_zip': '',
'same_instance_id': 1,
'same_os': 1,
'same_provider': 1,
'same_region': 1,
'same_service': 1,
'test_started': '2020-10-30 00:33:51',
'test_stopped': '2020-10-30 00:34:01',
'transfer': 11605.6
}, {
'bandwidth_direction': 'down',
'bandwidth_max': 9910.723,
'bandwidth_mean': 9730.44,
'bandwidth_median': 9866.29,
'bandwidth_min': 8349.13,
'bandwidth_p10': 8349.13,
'bandwidth_p25': 9888.001,
'bandwidth_p75': 9889.76,
'bandwidth_p90': 9889.8,
'bandwidth_stdev': 487,
'benchmark_version': 1.0,
'collectd_rrd': '',
'cpu_client': 34.346987,
'cpu_server': 2.880331,
'iperf_bandwidth': '',
'iperf_cmd': 'iperf3 -c 10.240.0.14 -J -i 1',
'iperf_concurrency': 1,
'iperf_interval': 1,
'iperf_len': '',
'iperf_mss': '',
'iperf_nodelay': '',
'iperf_num': '',
'iperf_parallel': 1,
'iperf_reverse': '',
'iperf_server': '10.240.0.14',
'iperf_server_instance_id': 'n1-standard-2',
'iperf_server_os': 'Ubuntu 18.04.5 LTS',
'iperf_server_provider': '',
'iperf_server_provider_id': 'google',
'iperf_server_region': 'us-central1',
'iperf_server_service': '',
'iperf_server_service_id': 'google:compute',
'iperf_time': 10,
'iperf_tos': '',
'iperf_udp': '',
'iperf_version': '3.1.3',
'iperf_warmup': 0,
'iperf_window': '',
'iperf_zerocopy': '',
'iteration': 1,
'jitter_max': '',
'jitter_mean': '',
'jitter_median': '',
'jitter_min': '',
'jitter_p10': '',
'jitter_p25': '',
'jitter_p75': '',
'jitter_p90': '',
'jitter_stdev': '',
'loss_max': '',
'loss_mean': '',
'loss_median': '',
'loss_min': '',
'loss_p10': '',
'loss_p25': '',
'loss_p75': '',
'loss_p90': '',
'loss_stdev': '',
'meta_compute_service': '',
'meta_compute_service_id': 'google:compute',
'meta_cpu': 'Intel Xeon 2.30GHz',
'meta_cpu_cache': '46080 KB',
'meta_cpu_cores': 2,
'meta_cpu_speed': 2300.0,
'meta_hostname': 'pkb-4c137ef3-0',
'meta_instance_id': 'n1-standard-2',
'meta_memory': '7 GB',
'meta_memory_gb': 7,
'meta_memory_mb': 7457,
'meta_os': 'Ubuntu 18.04.5 LTS',
'meta_provider': '',
'meta_provider_id': 'google',
'meta_region': 'us-central1',
'meta_resource_id': '',
'meta_run_group_id': '',
'meta_run_id': '',
'meta_test_id': '4c137ef3',
'report_pdf': '',
'report_zip': '',
'same_instance_id': 1,
'same_os': 1,
'same_provider': 1,
'same_region': 1,
'same_service': 1,
'test_started': '2021-02-25 4:20:12',
'test_stopped': '2021-02-25 4:22:01',
'transfer': 11605.5
}]
self.assertListEqual(expected_gartner_metadata, results)
def setUp(self):
super(GceNetworkTest, self).setUp()
# need a benchmarkspec in the context to run
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
'cluster_boot', flag_values=FLAGS)
benchmark_spec.BenchmarkSpec(mock.Mock(), config_spec, 'uid')
def _CreateBenchmarkSpec(self, benchmark_config_yaml):
config = configs.LoadConfig(benchmark_config_yaml, {}, NAME)
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
NAME, flag_values=FLAGS, **config)
return benchmark_spec.BenchmarkSpec(ping_benchmark, config_spec, UID)
def _CreateBenchmarkSpecFromConfigDict(self, config_dict, benchmark_name):
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
benchmark_name, flag_values=self._mocked_flags, **config_dict)
return benchmark_spec.BenchmarkSpec(config_spec, benchmark_name, UID)
def _CreateBenchmarkSpec(self, benchmark_config_yaml):
config = configs.LoadConfig(benchmark_config_yaml, {}, NAME)
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
NAME, flag_values=self._mocked_flags, **config)
return benchmark_spec.BenchmarkSpec(config_spec, NAME, UID)
def setUp(self):
# VM Creation depends on there being a BenchmarkSpec.
self.spec = benchmark_spec.BenchmarkSpec({}, 'name', 'benchmark_uid')
self.addCleanup(context.SetThreadBenchmarkSpec, None)
def RunBenchmark(benchmark, collector, sequence_number, total_benchmarks,
benchmark_config, benchmark_uid):
"""Runs a single benchmark and adds the results to the collector.
Args:
benchmark: The benchmark module to be run.
collector: The SampleCollector object to add samples to.
sequence_number: The sequence number of when the benchmark was started
relative to the other benchmarks in the suite.
total_benchmarks: The total number of benchmarks in the suite.
benchmark_config: The config to run the benchmark with.
benchmark_uid: An identifier unique to this run of the benchmark even
if the same benchmark is run multiple times with different configs.
"""
benchmark_name = benchmark.BENCHMARK_NAME
# Modify the logger prompt for messages logged within this function.
label_extension = '{}({}/{})'.format(benchmark_name, sequence_number,
total_benchmarks)
log_context = log_util.GetThreadLogContext()
with log_context.ExtendLabel(label_extension):
# Optional prerequisite checking.
check_prereqs = getattr(benchmark, 'CheckPrerequisites', None)
if check_prereqs:
try:
check_prereqs()
except:
logging.exception('Prerequisite check failed for %s',
benchmark_name)
raise
end_to_end_timer = timing_util.IntervalTimer()
detailed_timer = timing_util.IntervalTimer()
spec = None
try:
with end_to_end_timer.Measure('End to End'):
if FLAGS.run_stage in [STAGE_ALL, STAGE_PROVISION]:
# It is important to create the spec outside of DoProvisionPhase
# because if DoPreparePhase raises an exception, we still need
# a reference to the spec in order to delete it in the "finally"
# section below.
spec = benchmark_spec.BenchmarkSpec(
benchmark_config, benchmark_name, benchmark_uid)
spec.ConstructVirtualMachines()
DoProvisionPhase(benchmark_name, spec, detailed_timer)
else:
try:
spec = benchmark_spec.BenchmarkSpec.GetSpecFromFile(
benchmark_uid)
except IOError:
if FLAGS.run_stage == STAGE_PREPARE:
logging.error(
'We were unable to load the BenchmarkSpec. This may be '
'related to two additional run stages which have recently '
'been added. Please make sure to run the stage "provision" '
'before "prepare". Similarly, make sure to run "teardown" '
'after "cleanup".')
raise
if FLAGS.run_stage in [STAGE_ALL, STAGE_PREPARE]:
DoPreparePhase(benchmark, benchmark_name, spec,
detailed_timer)
if FLAGS.run_stage in [STAGE_ALL, STAGE_RUN]:
DoRunPhase(benchmark, benchmark_name, spec, collector,
detailed_timer)
if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]:
DoCleanupPhase(benchmark, benchmark_name, spec,
detailed_timer)
if FLAGS.run_stage in [STAGE_ALL, STAGE_TEARDOWN]:
DoTeardownPhase(benchmark_name, spec, detailed_timer)
# Add samples for any timed interval that was measured.
include_end_to_end = timing_util.EndToEndRuntimeMeasurementEnabled(
)
include_runtimes = timing_util.RuntimeMeasurementsEnabled()
include_timestamps = timing_util.TimestampMeasurementsEnabled()
if FLAGS.run_stage == STAGE_ALL:
collector.AddSamples(
end_to_end_timer.GenerateSamples(
include_runtime=include_end_to_end or include_runtimes,
include_timestamps=include_timestamps), benchmark_name,
spec)
collector.AddSamples(
detailed_timer.GenerateSamples(include_runtimes,
include_timestamps),
benchmark_name, spec)
except:
# Resource cleanup (below) can take a long time. Log the error to give
# immediate feedback, then re-throw.
logging.exception('Error during benchmark %s', benchmark_name)
# If the particular benchmark requests us to always call cleanup, do it
# here.
if (FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP] and spec
and spec.always_call_cleanup):
DoCleanupPhase(benchmark, benchmark_name, spec, detailed_timer)
raise
finally:
if spec:
if FLAGS.run_stage in [STAGE_ALL, STAGE_TEARDOWN]:
spec.Delete()
# Pickle spec to save final resource state.
spec.PickleSpec()
def makeSpec(self, yaml_benchmark_config=ping_benchmark.BENCHMARK_CONFIG):
config = configs.LoadConfig(yaml_benchmark_config, {}, NAME)
spec = benchmark_spec.BenchmarkSpec(config, NAME, UID)
spec.ConstructVirtualMachines()
return spec
def setUp(self):
self.addCleanup(context.SetThreadBenchmarkSpec, None)
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
_BENCHMARK_NAME, flag_values=mock_flags.MockFlags(), vm_groups={})
self.benchmark_spec = benchmark_spec.BenchmarkSpec(
mock.MagicMock(), config_spec, _BENCHMARK_UID)
def RunBenchmark(benchmark, collector, sequence_number, total_benchmarks):
"""Runs a single benchmark and adds the results to the collector.
Args:
benchmark: The benchmark module to be run.
collector: The SampleCollector object to add samples to.
sequence_number: The sequence number of when the benchmark was started
relative to the other benchmarks in the suite.
total_benchmarks: The total number of benchmarks in the suite.
"""
benchmark_info = benchmark.GetInfo()
if not ValidateBenchmarkInfo(benchmark_info):
return
benchmark_name = benchmark_info['name']
# Modify the logger prompt for messages logged within this function.
label_extension = '{}({}/{})'.format(benchmark_name, sequence_number,
total_benchmarks)
log_context = log_util.GetThreadLogContext()
with log_context.ExtendLabel(label_extension):
# Optional prerequisite checking.
check_prereqs = getattr(benchmark, 'CheckPrerequisites', None)
if check_prereqs:
try:
check_prereqs()
except:
logging.exception('Prerequisite check failed for %s',
benchmark_name)
raise
end_to_end_timer = timing_util.IntervalTimer()
detailed_timer = timing_util.IntervalTimer()
spec = None
try:
with end_to_end_timer.Measure('End to End'):
if FLAGS.run_stage in [STAGE_ALL, STAGE_PREPARE]:
# It is important to create the spec outside of DoPreparePhase
# because if DoPreparePhase raises an exception, we still need
# a reference to the spec in order to delete it in the "finally"
# section below.
spec = benchmark_spec.BenchmarkSpec(benchmark_info)
DoPreparePhase(benchmark, benchmark_name, spec,
detailed_timer)
else:
spec = benchmark_spec.BenchmarkSpec.GetSpecFromFile(
benchmark_name)
if FLAGS.run_stage in [STAGE_ALL, STAGE_RUN]:
DoRunPhase(benchmark, benchmark_name, spec, collector,
detailed_timer)
if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]:
DoCleanupPhase(benchmark, benchmark_name, spec,
detailed_timer)
# Add samples for any timed interval that was measured.
include_end_to_end = timing_util.EndToEndRuntimeMeasurementEnabled(
)
include_runtimes = timing_util.RuntimeMeasurementsEnabled()
include_timestamps = timing_util.TimestampMeasurementsEnabled()
if FLAGS.run_stage == STAGE_ALL:
collector.AddSamples(
end_to_end_timer.GenerateSamples(
include_runtime=include_end_to_end or include_runtimes,
include_timestamps=include_timestamps), benchmark_name,
spec)
collector.AddSamples(
detailed_timer.GenerateSamples(include_runtimes,
include_timestamps),
benchmark_name, spec)
except Exception:
# Resource cleanup (below) can take a long time. Log the error to give
# immediate feedback, then re-throw.
logging.exception('Error during benchmark %s', benchmark_name)
# If the particular benchmark requests us to always call cleanup, do it
# here.
if (FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP] and spec
and spec.always_call_cleanup):
DoCleanupPhase(benchmark, benchmark_name, spec, detailed_timer)
raise
finally:
if spec:
if FLAGS.run_stage in [STAGE_ALL, STAGE_CLEANUP]:
spec.Delete()
# Pickle spec to save final resource state.
spec.PickleSpec()
def _CreateBenchmarkSpecFromConfigDict(self, config_dict, benchmark_name):
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
benchmark_name, flag_values=FLAGS, **config_dict)
benchmark_module = next((b for b in linux_benchmarks.BENCHMARKS
if b.BENCHMARK_NAME == benchmark_name))
return benchmark_spec.BenchmarkSpec(benchmark_module, config_spec, UID)
def _CreateBenchmarkSpecFromConfigDict(self, config_dict, benchmark_name):
config_spec = benchmark_config_spec.BenchmarkConfigSpec(
benchmark_name, flag_values=FLAGS, **config_dict)
return benchmark_spec.BenchmarkSpec(mock.MagicMock(), config_spec, UID)
def testBackgroundWorkloadConfigBadIp(self):
""" Check that the config with invalid ip type gets an error """
config = configs.LoadConfig(CONFIG_WITH_BACKGROUND_NETWORK_BAD_IPFLAG,
{}, NAME)
with self.assertRaises(errors.Config.InvalidValue):
benchmark_spec.BenchmarkSpec(config, NAME, UID)