def testBenchmarkSupportFlag(self):
""" Test the benchmark_compatibility_checking flag
We use Kubernetes as our test cloud platform because it has
supported benchmarks (IsBenchmarkSupported returns true)
unsupported benchmarks (IsBenchmarkSupported returns false)
and returns None if the benchmark isn't in either list.
"""
with mock_flags.PatchFlags(self._mocked_flags):
self._mocked_flags.cloud = 'Kubernetes'
config = configs.LoadConfig(iperf_benchmark.BENCHMARK_CONFIG, {},
ALWAYS_SUPPORTED)
self.assertTrue(self.createBenchmarkSpec(config, ALWAYS_SUPPORTED))
with self.assertRaises(ValueError):
self.createBenchmarkSpec(config, NEVER_SUPPORTED)
self._mocked_flags.benchmark_compatibility_checking = 'permissive'
self.assertTrue(self.createBenchmarkSpec(config, ALWAYS_SUPPORTED),
'benchmark is supported, mode is permissive')
with self.assertRaises(ValueError):
self.createBenchmarkSpec(config, NEVER_SUPPORTED)
self._mocked_flags.benchmark_compatibility_checking = 'none'
self.assertTrue(self.createBenchmarkSpec(config, ALWAYS_SUPPORTED))
self.assertTrue(self.createBenchmarkSpec(config, NEVER_SUPPORTED))
def patch_critical_objects(stdout='',
stderr='',
return_code=0,
flags=mock_flags.MockFlags()):
with contextlib2.ExitStack() as stack:
flags.gcloud_path = 'gcloud'
flags.run_uri = _RUN_URI
flags.data_search_paths = ''
stack.enter_context(mock_flags.PatchFlags(flags))
stack.enter_context(mock.patch('__builtin__.open'))
stack.enter_context(mock.patch(vm_util.__name__ + '.PrependTempDir'))
stack.enter_context(
mock.patch(vm_util.__name__ + '.NamedTemporaryFile'))
stack.enter_context(
mock.patch(util.__name__ + '.GetDefaultProject',
return_value='fakeproject'))
stack.enter_context(
mock.patch(util.__name__ + '.GetDefaultUser',
return_value='fakeuser'))
retval = (stdout, stderr, return_code)
issue_command = stack.enter_context(
mock.patch(vm_util.__name__ + '.IssueCommand',
return_value=retval))
yield issue_command
def patch_critical_objects(stdout='', stderr='', return_code=0):
with contextlib2.ExitStack() as stack:
retval = (stdout, stderr, return_code)
mflags = mock_flags.MockFlags()
mflags.gcloud_path = 'gcloud'
mflags.run_uri = _RUN_URI
mflags.kubectl = _KUBECTL
mflags.kubeconfig = _KUBECONFIG
stack.enter_context(mock_flags.PatchFlags(mflags))
stack.enter_context(mock.patch('__builtin__.open'))
stack.enter_context(mock.patch(vm_util.__name__ + '.PrependTempDir'))
# Save and return the temp_file mock here so that we can access the write()
# call on the instance that the mock returned. This allows us to verify
# that the body of the file is what we expect it to be (useful for
# verifying that the pod.yml body was written correctly).
temp_file = stack.enter_context(
mock.patch(vm_util.__name__ + '.NamedTemporaryFile'))
issue_command = stack.enter_context(
mock.patch(vm_util.__name__ + '.IssueCommand',
return_value=retval))
yield issue_command, temp_file
def testBackgroundWorkloadSpec(self):
""" Check that the benchmark spec calls the prepare, stop, and start
methods on the vms """
with mock_flags.PatchFlags() as mocked_flags:
self.setupCommonFlags(mocked_flags)
mocked_flags.background_cpu_threads = 1
collector = mock.MagicMock()
config = configs.LoadConfig(ping_benchmark.BENCHMARK_CONFIG, {},
NAME)
spec = benchmark_spec.BenchmarkSpec(config, NAME, UID)
vm0 = mock.MagicMock()
vm1 = mock.MagicMock()
spec.ConstructVirtualMachines()
spec.vms = [vm0, vm1]
timer = timing_util.IntervalTimer()
pkb.DoPreparePhase(ping_benchmark, NAME, spec, timer)
for vm in spec.vms:
self.assertEqual(vm.PrepareBackgroundWorkload.call_count, 1)
with mock.patch(ping_benchmark.__name__ + '.Run'):
ping_benchmark.Run.side_effect = functools.partial(
self._CheckAndIncrement, expected_last_call=1)
vm0.StartBackgroundWorkload.side_effect = functools.partial(
self._CheckAndIncrement, expected_last_call=0)
vm0.StopBackgroundWorkload.side_effect = functools.partial(
self._CheckAndIncrement, expected_last_call=2)
pkb.DoRunPhase(ping_benchmark, NAME, spec, collector, timer)
self.assertEqual(ping_benchmark.Run.call_count, 1)
for vm in spec.vms:
self.assertEqual(vm.StartBackgroundWorkload.call_count, 1)
self.assertEqual(vm.StopBackgroundWorkload.call_count, 1)
def testZonesFlag(self):
with mock_flags.PatchFlags(self._mocked_flags):
self._mocked_flags.zones = ['us-east-1b', 'zone2']
spec = self._CreateBenchmarkSpecFromYaml(MULTI_CLOUD_CONFIG)
spec.ConstructVirtualMachines()
self.assertEqual(len(spec.vms), 2)
self.assertEqual(spec.vm_groups['group1'][0].zone, 'us-east-1b')
self.assertEqual(spec.vm_groups['group2'][0].zone, 'zone2')
def CreateManagedDbFromSpec(spec_dict):
mock_db_spec = mock.Mock(
spec=benchmark_config_spec._ManagedRelationalDbSpec)
mock_db_spec.configure_mock(**spec_dict)
mocked_flags = mock_flags.MockFlags()
mocked_flags.run_uri = 'mock-run-uri'
with mock_flags.PatchFlags(mocked_flags):
db_class = gcp_managed_relational_db.GCPManagedRelationalDb(mock_db_spec)
return db_class
def testGetFlag(self):
self.flags['test_flag'].parse(5)
with mock_flags.PatchFlags(self.flags):
self.assertEqual(FLAGS.test_flag, 5)
self.assertTrue(FLAGS['test_flag'].present)
self.assertEqual(FLAGS['test_flag'].value, 5)
self.assertEqual(FLAGS.test_flag, 0)
self.assertFalse(FLAGS['test_flag'].present)
self.assertEqual(FLAGS['test_flag'].value, 0)
def testBackgroundWorkloadVM(self):
""" Check that the vm background workload calls work """
with mock_flags.PatchFlags() as mocked_flags:
self.setupCommonFlags(mocked_flags)
mocked_flags.background_cpu_threads = 1
config = configs.LoadConfig(ping_benchmark.BENCHMARK_CONFIG, {},
NAME)
spec = benchmark_spec.BenchmarkSpec(config, NAME, UID)
spec.ConstructVirtualMachines()
self._CheckVMFromSpec(spec, 2)
def _PatchCriticalObjects(self):
"""A context manager that patches a few critical objects with mocks."""
with mock.patch(vm_util.__name__ + '.IssueCommand') as issue_command, \
mock.patch('__builtin__.open'), \
mock.patch(vm_util.__name__ + '.NamedTemporaryFile'), \
mock_flags.PatchFlags() as mocked_flags:
mocked_flags.gcloud_path = 'test_gcloud'
mocked_flags.gcloud_scopes = None
mocked_flags.run_uri = 'aaaaaa'
yield issue_command, mocked_flags
def setUp(self):
mocked_flags = mock_flags.MockFlags()
mocked_flags.project = 'project'
mock_spec = mock.Mock(spec=benchmark_config_spec._CloudRedisSpec)
mock_spec.redis_name = 'foobar'
mock_spec.redis_tier = 'tier'
mock_spec.redis_size_gb = 5
mock_spec.redis_version = 'version'
with mock_flags.PatchFlags(mocked_flags):
self.redis = gcp_cloud_redis.CloudRedis(mock_spec)
def testCopyPatchedFlags(self):
with mock_flags.PatchFlags(self.flags):
FLAGS.test_flag = 1
copied_flags = copy.deepcopy(FLAGS)
copied_flags.test_flag = 2
self.assertEqual(FLAGS.test_flag, 1)
self.assertEqual(self.flags.test_flag, 1)
self.assertEqual(copied_flags.test_flag, 2)
self.assertEqual(FLAGS.test_flag, 0)
self.assertEqual(self.flags.test_flag, 1)
self.assertEqual(copied_flags.test_flag, 2)
def testBackgroundWorkloadConfig(self):
""" Check that the config can be used to set the background cpu threads """
with mock_flags.PatchFlags() as mocked_flags:
self.setupCommonFlags(mocked_flags)
config = configs.LoadConfig(CONFIG_WITH_BACKGROUND_CPU, {}, NAME)
spec = benchmark_spec.BenchmarkSpec(config, NAME, UID)
spec.ConstructVirtualMachines()
for vm in spec.vm_groups['vm_1']:
self.assertIsNone(vm.background_cpu_threads)
for vm in spec.vm_groups['vm_2']:
self.assertEqual(vm.background_cpu_threads, 3)
self._CheckVMFromSpec(spec, 1)
def testBackgroundWorkloadVanillaConfigFlag(self):
""" Check that the flag overrides the config """
with mock_flags.PatchFlags() as mocked_flags:
self.setupCommonFlags(mocked_flags)
mocked_flags.background_cpu_threads = 2
config = configs.LoadConfig(ping_benchmark.BENCHMARK_CONFIG, {},
NAME)
spec = benchmark_spec.BenchmarkSpec(config, NAME, UID)
spec.ConstructVirtualMachines()
for vm in spec.vms:
self.assertEqual(vm.background_cpu_threads, 2)
self._CheckVMFromSpec(spec, 2)
def setUp(self):
mocked_flags = mock_flags.MockFlags()
mocked_flags.project = 'project'
mocked_flags.redis_failover_style = cloud_redis.Failover.FAILOVER_NONE
mock_spec = mock.Mock(spec=benchmark_config_spec._CloudRedisSpec)
mock_spec.redis_name = 'foobar'
mock_spec.redis_tier = 'tier'
mock_spec.redis_size_gb = 5
mock_spec.redis_version = 'version'
mock_spec.client_vm = mock.Mock()
with mock_flags.PatchFlags(mocked_flags):
self.redis = gcp_cloud_redis.CloudRedis(mock_spec)
def testBackgroundWorkloadVanillaConfig(self):
""" Test that nothing happens with the vanilla config """
with mock_flags.PatchFlags() as mocked_flags:
self.setupCommonFlags(mocked_flags)
config = configs.LoadConfig(ping_benchmark.BENCHMARK_CONFIG, {},
NAME)
spec = benchmark_spec.BenchmarkSpec(config, NAME, UID)
spec.ConstructVirtualMachines()
for vm in spec.vms:
self.assertIsNone(vm.background_cpu_threads)
self.assertIsNone(vm.background_network_mbits_per_sec)
self._CheckVMFromSpec(spec, 0)
def testSetFlag(self):
with mock_flags.PatchFlags(self.flags):
FLAGS.test_flag = 5
self.assertEqual(FLAGS.test_flag, 5)
self.assertFalse(FLAGS['test_flag'].present)
self.assertEqual(FLAGS['test_flag'].value, 5)
FLAGS['test_flag'].present = True
self.assertTrue(FLAGS['test_flag'].present)
self.assertEqual(FLAGS['test_flag'].value, 5)
self.assertEqual(self.flags.test_flag, 5)
self.assertTrue(self.flags['test_flag'].present)
self.assertEqual(self.flags['test_flag'].value, 5)
self.assertEqual(FLAGS.test_flag, 0)
self.assertFalse(FLAGS['test_flag'].present)
self.assertEqual(FLAGS['test_flag'].value, 0)
def testWindowsVMCausesError(self):
""" windows vm with background_cpu_threads raises exception """
with mock_flags.PatchFlags() as mocked_flags:
self.setupCommonFlags(mocked_flags)
mocked_flags.background_cpu_threads = 1
mocked_flags.os_type = benchmark_spec.WINDOWS
config = configs.LoadConfig(ping_benchmark.BENCHMARK_CONFIG, {},
NAME)
spec = benchmark_spec.BenchmarkSpec(config, NAME, UID)
spec.ConstructVirtualMachines()
with self.assertRaises(Exception):
spec.Prepare()
with self.assertRaises(Exception):
spec.StartBackgroundWorkload()
with self.assertRaises(Exception):
spec.StopBackgroundWorkload()
def testEMRConfig(self):
self._mocked_flags.cloud = providers.AWS
self._mocked_flags.zones = 'us-west-2'
with mock_flags.PatchFlags(self._mocked_flags):
spec = self._CreateBenchmarkSpecFromYaml(SERVICE_CONFIG)
spec.ConstructSparkService()
spec.ConstructVirtualMachines()
self.assertTrue(hasattr(spec, 'spark_service'))
self.assertTrue(spec.spark_service is not None)
self.assertEqual(len(spec.vms), 0)
self.assertEqual(spec.config.spark_service.worker_group.vm_count,
4, str(spec.config.spark_service.__dict__))
machine_type = spec.config.spark_service.worker_group.vm_spec.machine_type
self.assertEqual(spec.config.spark_service.service_type,
spark_service.PROVIDER_MANAGED)
self.assertEqual(machine_type, 'm4.xlarge',
str(spec.config.spark_service.__dict__))
self.assertTrue(isinstance(spec.spark_service, aws_emr.AwsEMR))
def testFilename(self):
"""Tests the objects written filename can be parsed for an age."""
with mock_flags.PatchFlags() as mocked_flags:
mocked_flags.object_storage_region = 'us-central1-a'
mocked_flags.object_storage_objects_written_file_prefix = 'prefix'
write_time = datetime.datetime.now() - datetime.timedelta(hours=72)
with mock.patch.object(object_storage_service_benchmark, '_DatetimeNow',
return_value=write_time):
filename = (
object_storage_service_benchmark._ColdObjectsWrittenFilename())
read_time = datetime.datetime.now()
with mock.patch.object(object_storage_service_benchmark, '_DatetimeNow',
return_value=read_time):
age = object_storage_service_benchmark._ColdObjectsWrittenFileAgeHours(
filename)
# Verify that the age is between 72 and 73 hours.
self.assertLessEqual(72, age)
self.assertLessEqual(age, 73)
def testBenchmarkConfigSpecLoadsProvider(self):
p = mock.patch(providers.__name__ + '.LoadProvider')
p.start()
self.addCleanup(p.stop)
config = {
'vm_groups': {
'group1': {
'cloud': 'AWS',
'os_type': 'debian',
'vm_count': 0,
'vm_spec': {'AWS': {}}
}
}
}
with mock_flags.PatchFlags() as mocked_flags:
benchmark_config_spec.BenchmarkConfigSpec(
'name', flag_values=mocked_flags, **config)
providers.LoadProvider.assert_called_with('aws')
