def test_execute_schedule_throughput_throttled(self, es): es.bulk.return_value = {"errors": False} params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", description="unittest track", indices=None, challenges=None) # in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples for target_throughput, bounds in { 10: [2, 4], 100: [24, 26], 1000: [245, 255] }.items(): task = track.Task("time-based", track.Operation( "time-based", track.OperationType.Bulk.name, params={ "body": ["action_metadata_line", "index_line"], "action_metadata_present": True, "bulk-size": 1 }, param_source="driver-test-param-source"), warmup_time_period=0.5, time_period=0.5, clients=4, params={ "target-throughput": target_throughput, "clients": 4 }, completes_parent=True) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=0, task=task, start_timestamp=0) cancel = threading.Event() complete = threading.Event() execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete) execute_schedule() samples = sampler.samples sample_size = len(samples) lower_bound = bounds[0] upper_bound = bounds[1] self.assertTrue( lower_bound <= sample_size <= upper_bound, msg="Expected sample size to be between %d and %d but was %d" % (lower_bound, upper_bound, sample_size)) self.assertTrue( complete.is_set(), "Executor should auto-complete a task that terminates its parent" )
def test_execute_schedule_aborts_on_error(self, es): class ExpectedUnitTestException(Exception): pass def run(*args, **kwargs): raise ExpectedUnitTestException() task = track.Task("no-op", track.Operation( "no-op", track.OperationType.Bulk.name, params={}, param_source="driver-test-param-source"), warmup_time_period=0.5, time_period=0.5, clients=4, params={"clients": 4}) schedule = [(0, metrics.SampleType.Warmup, 0, self.context_managed(run), None)] sampler = driver.Sampler(client_id=0, task=None, start_timestamp=0) cancel = threading.Event() complete = threading.Event() execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete) with self.assertRaises(ExpectedUnitTestException): execute_schedule() es.assert_not_called()
def test_cancel_execute_schedule(self, es): es.bulk.return_value = { "errors": False } params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", short_description="unittest track", source_root_url="http://example.org", indices=None, challenges=None) # in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples for target_throughput, bounds in {10: [2, 4], 100: [24, 26], 1000: [245, 255]}.items(): task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={ "body": ["action_metadata_line", "index_line"], "action_metadata_present": True, "bulk-size": 1 }, param_source="driver-test-param-source"), warmup_time_period=0.5, time_period=0.5, clients=4, params={"target-throughput": target_throughput, "clients": 4}) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=0, task=task, start_timestamp=0) cancel = threading.Event() complete = threading.Event() execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete) cancel.set() execute_schedule() samples = sampler.samples sample_size = len(samples) self.assertEqual(0, sample_size)
def test_execute_schedule_throughput_throttled(self, es): es.bulk.return_value = { "errors": False } params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", short_description="unittest track", description="unittest track", source_root_url="http://example.org", indices=None, challenges=None) # in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples for target_throughput, bounds in {10: [2, 4], 100: [24, 26], 1000: [245, 255]}.items(): task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={ "body": ["action_metadata_line", "index_line"], "action_metadata_present": True }, param_source="driver-test-param-source"), warmup_time_period=0.5, time_period=0.5, clients=4, target_throughput=target_throughput) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=0, operation=task.operation, start_timestamp=0) driver.execute_schedule(schedule, es, sampler) samples = sampler.samples sample_size = len(samples) lower_bound = bounds[0] upper_bound = bounds[1] self.assertTrue(lower_bound <= sample_size <= upper_bound, msg="Expected sample size to be between %d and %d but was %d" % (lower_bound, upper_bound, sample_size))
def test_execute_schedule_in_throughput_mode(self, es): es.bulk.return_value = {"errors": False} params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", description="unittest track", indices=None, challenges=None) task = track.Task( "time-based", track.Operation( "time-based", track.OperationType.Bulk.name, params={ "body": ["action_metadata_line", "index_line"], "action-metadata-present": True, "bulk-size": 1, # we need this because DriverTestParamSource does not know that we only have one bulk and hence size() returns incorrect results "size": 1 }, param_source="driver-test-param-source"), warmup_time_period=0, clients=4) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=2, task=task, start_timestamp=time.perf_counter()) cancel = threading.Event() complete = threading.Event() execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete) execute_schedule() samples = sampler.samples self.assertTrue(len(samples) > 0) self.assertFalse(complete.is_set(), "Executor should not auto-complete a normal task") previous_absolute_time = -1.0 previous_relative_time = -1.0 for sample in samples: self.assertEqual(2, sample.client_id) self.assertEqual(task, sample.task) self.assertLess(previous_absolute_time, sample.absolute_time) previous_absolute_time = sample.absolute_time self.assertLess(previous_relative_time, sample.relative_time) previous_relative_time = sample.relative_time # we don't have any warmup time period self.assertEqual(metrics.SampleType.Normal, sample.sample_type) # latency equals service time in throughput mode self.assertEqual(sample.latency_ms, sample.service_time_ms) self.assertEqual(1, sample.total_ops) self.assertEqual("docs", sample.total_ops_unit) self.assertEqual(1, sample.request_meta_data["bulk-size"])
def test_execute_schedule_aborts_on_error(self, es): class ExpectedUnitTestException(Exception): pass def run(*args, **kwargs): raise ExpectedUnitTestException() schedule = [(0, metrics.SampleType.Warmup, 0, self.context_managed(run), None)] sampler = driver.Sampler(client_id=0, operation=None, start_timestamp=0) with self.assertRaises(ExpectedUnitTestException): driver.execute_schedule(schedule, es, sampler=sampler) es.assert_not_called()
def test_execute_schedule_in_throughput_mode(self, es): es.bulk.return_value = {"errors": False} params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", short_description="unittest track", description="unittest track", source_root_url="http://example.org", indices=None, challenges=None) task = track.Task(track.Operation( "time-based", track.OperationType.Index.name, params={ "body": ["action_metadata_line", "index_line"], "action_metadata_present": True }, param_source="driver-test-param-source"), warmup_time_period=0, clients=4, target_throughput=None) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=2, task=task, start_timestamp=100) cancel = threading.Event() driver.execute_schedule(cancel, 0, task.operation, schedule, es, sampler) samples = sampler.samples self.assertTrue(len(samples) > 0) previous_absolute_time = -1.0 previous_relative_time = -1.0 for sample in samples: self.assertEqual(2, sample.client_id) self.assertEqual(task, sample.task) self.assertTrue(previous_absolute_time < sample.absolute_time) previous_absolute_time = sample.absolute_time self.assertTrue(previous_relative_time < sample.relative_time) previous_relative_time = sample.relative_time # we don't have any warmup time period self.assertEqual(metrics.SampleType.Normal, sample.sample_type) # latency equals service time in throughput mode self.assertEqual(sample.latency_ms, sample.service_time_ms) self.assertEqual(1, sample.total_ops) self.assertEqual("docs", sample.total_ops_unit) self.assertEqual(1, sample.request_meta_data["bulk-size"])
def test_execute_schedule_with_progress_determined_by_runner(self, es): es.bulk.return_value = { "errors": False } params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", description="unittest track", indices=None, challenges=None) task = track.Task("time-based", track.Operation("time-based", operation_type="unit-test-recovery", params={ "indices-to-restore": "*", # The runner will determine progress "size": None }, param_source="driver-test-param-source"), warmup_time_period=0, clients=4) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=2, task=task, start_timestamp=time.perf_counter()) cancel = threading.Event() complete = threading.Event() execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete) execute_schedule() samples = sampler.samples self.assertEqual(5, len(samples)) self.assertTrue(self.runner_with_progress.completed) self.assertEqual(1.0, self.runner_with_progress.percent_completed) self.assertFalse(complete.is_set(), "Executor should not auto-complete a normal task") previous_absolute_time = -1.0 previous_relative_time = -1.0 for sample in samples: self.assertEqual(2, sample.client_id) self.assertEqual(task, sample.task) self.assertLess(previous_absolute_time, sample.absolute_time) previous_absolute_time = sample.absolute_time self.assertLess(previous_relative_time, sample.relative_time) previous_relative_time = sample.relative_time # we don't have any warmup time period self.assertEqual(metrics.SampleType.Normal, sample.sample_type) # latency equals service time in throughput mode self.assertEqual(sample.latency_ms, sample.service_time_ms) self.assertEqual(1, sample.total_ops) self.assertEqual("ops", sample.total_ops_unit)
async def _run_benchmark(self): # avoid: aiohttp.internal WARNING The object should be created from async function es = self.create_es_clients(sync=False) try: cancel = threading.Event() # allow to buffer more events than by default as we expect to have way more clients. self.sampler = driver.Sampler(start_timestamp=time.perf_counter(), buffer_size=65536) for task in self.challenge.schedule: self.current_tasks = [] aws = [] for sub_task in task: self.current_tasks.append(sub_task) self.logger.info("Running task [%s] with [%d] clients...", sub_task.name, sub_task.clients) for client_id in range(sub_task.clients): schedule = driver.schedule_for(self.track, sub_task, client_id) # used to indicate that we want to prematurely consider this completed. This is *not* due to # cancellation but a regular event in a benchmark and used to model task dependency of parallel tasks. complete = threading.Event() e = driver.AsyncExecutor(client_id, sub_task, schedule, es, self.sampler, cancel, complete, self.abort_on_error) aws.append(e()) # join point _ = await asyncio.gather(*aws) self.logger.info("All clients have finished running task [%s]", task.name) # drain the active samples before we move on to the next task self.update_samples() self.post_process_samples() self.reset_relative_time() self.update_progress_message(task_finished=True) finally: await asyncio.get_event_loop().shutdown_asyncgens() for e in es.values(): await e.transport.close()