def test_execute_schedule_throughput_throttled(self, es): es.bulk.return_value = { "errors": False } params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", short_description="unittest track", description="unittest track", source_root_url="http://example.org", indices=None, challenges=None) # in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples for target_throughput, bounds in {10: [2, 4], 100: [24, 26], 1000: [245, 255]}.items(): task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={ "body": ["action_metadata_line", "index_line"], "action_metadata_present": True }, param_source="driver-test-param-source"), warmup_time_period=0.5, time_period=0.5, clients=4, target_throughput=target_throughput) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=0, operation=task.operation, start_timestamp=0) driver.execute_schedule(schedule, es, sampler) samples = sampler.samples sample_size = len(samples) lower_bound = bounds[0] upper_bound = bounds[1] self.assertTrue(lower_bound <= sample_size <= upper_bound, msg="Expected sample size to be between %d and %d but was %d" % (lower_bound, upper_bound, sample_size))
def test_search_task_one_client(self): task = track.Task("search", track.Operation( "search", track.OperationType.Search.name, param_source="driver-test-param-source"), warmup_iterations=3, iterations=5, clients=1, params={ "target-throughput": 10, "clients": 1 }) schedule = driver.schedule_for(self.test_track, task, 0) expected_schedule = [ (0, metrics.SampleType.Warmup, 1 / 8, {}), (0.1, metrics.SampleType.Warmup, 2 / 8, {}), (0.2, metrics.SampleType.Warmup, 3 / 8, {}), (0.3, metrics.SampleType.Normal, 4 / 8, {}), (0.4, metrics.SampleType.Normal, 5 / 8, {}), (0.5, metrics.SampleType.Normal, 6 / 8, {}), (0.6, metrics.SampleType.Normal, 7 / 8, {}), (0.7, metrics.SampleType.Normal, 8 / 8, {}), ] self.assert_schedule(expected_schedule, schedule)
def test_cancel_execute_schedule(self, es): es.bulk.return_value = { "errors": False } params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", short_description="unittest track", source_root_url="http://example.org", indices=None, challenges=None) # in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples for target_throughput, bounds in {10: [2, 4], 100: [24, 26], 1000: [245, 255]}.items(): task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={ "body": ["action_metadata_line", "index_line"], "action_metadata_present": True, "bulk-size": 1 }, param_source="driver-test-param-source"), warmup_time_period=0.5, time_period=0.5, clients=4, params={"target-throughput": target_throughput, "clients": 4}) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=0, task=task, start_timestamp=0) cancel = threading.Event() complete = threading.Event() execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete) cancel.set() execute_schedule() samples = sampler.samples sample_size = len(samples) self.assertEqual(0, sample_size)
def test_schedule_param_source_determines_iterations_no_warmup(self): # we neither define any time-period nor any iteration count on the task. task = track.Task("bulk-index", track.Operation( "bulk-index", track.OperationType.Bulk.name, params={ "body": ["a"], "size": 3 }, param_source="driver-test-param-source"), clients=1, params={ "target-throughput": 4, "clients": 4 }) invocations = driver.schedule_for(self.test_track, task, 0) self.assert_schedule([ (0.0, metrics.SampleType.Normal, 1 / 3, { "body": ["a"], "size": 3 }), (1.0, metrics.SampleType.Normal, 2 / 3, { "body": ["a"], "size": 3 }), (2.0, metrics.SampleType.Normal, 3 / 3, { "body": ["a"], "size": 3 }), ], list(invocations))
def test_eternal_schedule(self): task = track.Task("time-based", track.Operation( "time-based", track.OperationType.Bulk.name, params={"body": ["a"]}, param_source="driver-test-param-source"), warmup_time_period=0, clients=4, params={ "target-throughput": 4, "clients": 4 }) invocations = driver.schedule_for(self.test_track, task, 0) self.assert_schedule([ (0.0, metrics.SampleType.Normal, None, { "body": ["a"] }), (1.0, metrics.SampleType.Normal, None, { "body": ["a"] }), (2.0, metrics.SampleType.Normal, None, { "body": ["a"] }), (3.0, metrics.SampleType.Normal, None, { "body": ["a"] }), (4.0, metrics.SampleType.Normal, None, { "body": ["a"] }), ], invocations, eternal_schedule=True)
def test_schedule_for_time_based(self): task = track.Task("time-based", track.Operation("time-based", track.OperationType.Bulk.name, params={"body": ["a"], "size": 11}, param_source="driver-test-param-source"), warmup_time_period=0.1, time_period=0.1, clients=1) schedule_handle = driver.schedule_for(self.test_track, task, 0) schedule = list(schedule_handle()) self.assertTrue(len(schedule) > 0) last_progress = -1 for invocation_time, sample_type, progress_percent, runner, params in schedule: # we're not throughput throttled self.assertEqual(0, invocation_time) if progress_percent <= 0.5: self.assertEqual(metrics.SampleType.Warmup, sample_type) else: self.assertEqual(metrics.SampleType.Normal, sample_type) self.assertTrue(last_progress < progress_percent) last_progress = progress_percent self.assertTrue(round(progress_percent, 2) >= 0.0, "progress should be >= 0.0 but was [%f]" % progress_percent) self.assertTrue(round(progress_percent, 2) <= 1.0, "progress should be <= 1.0 but was [%f]" % progress_percent) self.assertIsNotNone(runner, "runner must be defined") self.assertEqual({"body": ["a"], "size": 11}, params)
def test_execute_schedule_throughput_throttled(self, es): es.bulk.return_value = {"errors": False} params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", description="unittest track", indices=None, challenges=None) # in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples for target_throughput, bounds in { 10: [2, 4], 100: [24, 26], 1000: [245, 255] }.items(): task = track.Task("time-based", track.Operation( "time-based", track.OperationType.Bulk.name, params={ "body": ["action_metadata_line", "index_line"], "action_metadata_present": True, "bulk-size": 1 }, param_source="driver-test-param-source"), warmup_time_period=0.5, time_period=0.5, clients=4, params={ "target-throughput": target_throughput, "clients": 4 }, completes_parent=True) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=0, task=task, start_timestamp=0) cancel = threading.Event() complete = threading.Event() execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete) execute_schedule() samples = sampler.samples sample_size = len(samples) lower_bound = bounds[0] upper_bound = bounds[1] self.assertTrue( lower_bound <= sample_size <= upper_bound, msg="Expected sample size to be between %d and %d but was %d" % (lower_bound, upper_bound, sample_size)) self.assertTrue( complete.is_set(), "Executor should auto-complete a task that terminates its parent" )
def test_execute_schedule_in_throughput_mode(self, es): es.bulk.return_value = {"errors": False} params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", description="unittest track", indices=None, challenges=None) task = track.Task( "time-based", track.Operation( "time-based", track.OperationType.Bulk.name, params={ "body": ["action_metadata_line", "index_line"], "action-metadata-present": True, "bulk-size": 1, # we need this because DriverTestParamSource does not know that we only have one bulk and hence size() returns incorrect results "size": 1 }, param_source="driver-test-param-source"), warmup_time_period=0, clients=4) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=2, task=task, start_timestamp=time.perf_counter()) cancel = threading.Event() complete = threading.Event() execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete) execute_schedule() samples = sampler.samples self.assertTrue(len(samples) > 0) self.assertFalse(complete.is_set(), "Executor should not auto-complete a normal task") previous_absolute_time = -1.0 previous_relative_time = -1.0 for sample in samples: self.assertEqual(2, sample.client_id) self.assertEqual(task, sample.task) self.assertLess(previous_absolute_time, sample.absolute_time) previous_absolute_time = sample.absolute_time self.assertLess(previous_relative_time, sample.relative_time) previous_relative_time = sample.relative_time # we don't have any warmup time period self.assertEqual(metrics.SampleType.Normal, sample.sample_type) # latency equals service time in throughput mode self.assertEqual(sample.latency_ms, sample.service_time_ms) self.assertEqual(1, sample.total_ops) self.assertEqual("docs", sample.total_ops_unit) self.assertEqual(1, sample.request_meta_data["bulk-size"])
def test_schedule_defaults_to_iteration_based(self): # no time-period and no iterations specified on the task. Also, the parameter source does not define a size. task = track.Task("bulk-index", track.Operation("bulk-index", track.OperationType.Bulk.name, params={"body": ["a"]}, param_source="driver-test-param-source"), clients=1, params={"target-throughput": 4, "clients": 4}) invocations = driver.schedule_for(self.test_track, task, 0) self.assert_schedule([ (0.0, metrics.SampleType.Normal, 1 / 1, {"body": ["a"]}), ], list(invocations))
def test_search_task_two_clients(self): task = track.Task(track.Operation("search", track.OperationType.Search.name, param_source="driver-test-param-source"), warmup_iterations=2, iterations=10, clients=2, params={"target-throughput": 10, "clients": 2}) schedule = driver.schedule_for(self.test_track, task, 0) expected_schedule = [ (0, metrics.SampleType.Warmup, 1 / 6, {}), (0.2, metrics.SampleType.Normal, 2 / 6, {}), (0.4, metrics.SampleType.Normal, 3 / 6, {}), (0.6, metrics.SampleType.Normal, 4 / 6, {}), (0.8, metrics.SampleType.Normal, 5 / 6, {}), (1.0, metrics.SampleType.Normal, 6 / 6, {}), ] self.assert_schedule(expected_schedule, schedule)
def test_schedule_param_source_determines_iterations_including_warmup(self): task = track.Task("bulk-index", track.Operation("bulk-index", track.OperationType.Bulk.name, params={"body": ["a"], "size": 5}, param_source="driver-test-param-source"), warmup_iterations=2, clients=1, params={"target-throughput": 4, "clients": 4}) invocations = driver.schedule_for(self.test_track, task, 0) self.assert_schedule([ (0.0, metrics.SampleType.Warmup, 1 / 5, {"body": ["a"], "size": 5}), (1.0, metrics.SampleType.Warmup, 2 / 5, {"body": ["a"], "size": 5}), (2.0, metrics.SampleType.Normal, 3 / 5, {"body": ["a"], "size": 5}), (3.0, metrics.SampleType.Normal, 4 / 5, {"body": ["a"], "size": 5}), (4.0, metrics.SampleType.Normal, 5 / 5, {"body": ["a"], "size": 5}), ], list(invocations))
def test_finite_schedule_with_progress_indication(self): task = track.Task("time-based", track.Operation("time-based", track.OperationType.Bulk.name, params={"body": ["a"], "size": 5}, param_source="driver-test-param-source"), warmup_time_period=0, clients=4, params={"target-throughput": 4, "clients": 4}) invocations = driver.schedule_for(self.test_track, task, 0) self.assert_schedule([ (0.0, metrics.SampleType.Normal, 1 / 5, {"body": ["a"], "size": 5}), (1.0, metrics.SampleType.Normal, 2 / 5, {"body": ["a"], "size": 5}), (2.0, metrics.SampleType.Normal, 3 / 5, {"body": ["a"], "size": 5}), (3.0, metrics.SampleType.Normal, 4 / 5, {"body": ["a"], "size": 5}), (4.0, metrics.SampleType.Normal, 5 / 5, {"body": ["a"], "size": 5}), ], list(invocations), infinite_schedule=False)
def test_search_task_two_clients(self): task = track.Task(track.Operation("search", track.OperationType.Search.name, param_source="driver-test-param-source"), warmup_iterations=2, iterations=10, clients=2, target_throughput=10) schedule = driver.schedule_for(self.test_track, task, 0) expected_schedule = [ (0, metrics.SampleType.Warmup, 0, 6, None, {}), (0.2, metrics.SampleType.Normal, 0, 6, None, {}), (0.4, metrics.SampleType.Normal, 1, 6, None, {}), (0.6, metrics.SampleType.Normal, 2, 6, None, {}), (0.8, metrics.SampleType.Normal, 3, 6, None, {}), (1.0, metrics.SampleType.Normal, 4, 6, None, {}), ] self.assert_schedule(expected_schedule, schedule)
def test_eternal_schedule(self): task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={"body": ["a"]}, param_source="driver-test-param-source"), warmup_time_period=0, clients=4, params={"target-throughput": 4, "clients": 4}) invocations = driver.schedule_for(self.test_track, task, 0) self.assert_schedule([ (0.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (1.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (2.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (3.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (4.0, metrics.SampleType.Normal, None, {"body": ["a"]}), ], invocations, eternal_schedule=True)
def test_execute_schedule_in_throughput_mode(self, es): es.bulk.return_value = { "errors": False } params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", short_description="unittest track", source_root_url="http://example.org", indices=None, challenges=None) task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={ "body": ["action_metadata_line", "index_line"], "action_metadata_present": True, "bulk-size": 1, # we need this because DriverTestParamSource does not know that we only have one bulk and hence size() returns incorrect results "size": 1 }, param_source="driver-test-param-source"), warmup_time_period=0, clients=4) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=2, task=task, start_timestamp=100) cancel = threading.Event() complete = threading.Event() execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete) execute_schedule() samples = sampler.samples self.assertTrue(len(samples) > 0) self.assertFalse(complete.is_set(), "Executor should not auto-complete a normal task") previous_absolute_time = -1.0 previous_relative_time = -1.0 for sample in samples: self.assertEqual(2, sample.client_id) self.assertEqual(task, sample.task) self.assertTrue(previous_absolute_time < sample.absolute_time) previous_absolute_time = sample.absolute_time self.assertTrue(previous_relative_time < sample.relative_time) previous_relative_time = sample.relative_time # we don't have any warmup time period self.assertEqual(metrics.SampleType.Normal, sample.sample_type) # latency equals service time in throughput mode self.assertEqual(sample.latency_ms, sample.service_time_ms) self.assertEqual(1, sample.total_ops) self.assertEqual("docs", sample.total_ops_unit) self.assertEqual(1, sample.request_meta_data["bulk-size"])
def test_execute_schedule_in_throughput_mode(self, es): es.bulk.return_value = {"errors": False} params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", short_description="unittest track", description="unittest track", source_root_url="http://example.org", indices=None, challenges=None) task = track.Task(track.Operation( "time-based", track.OperationType.Index.name, params={ "body": ["action_metadata_line", "index_line"], "action_metadata_present": True }, param_source="driver-test-param-source"), warmup_time_period=0, clients=4, target_throughput=None) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=2, task=task, start_timestamp=100) cancel = threading.Event() driver.execute_schedule(cancel, 0, task.operation, schedule, es, sampler) samples = sampler.samples self.assertTrue(len(samples) > 0) previous_absolute_time = -1.0 previous_relative_time = -1.0 for sample in samples: self.assertEqual(2, sample.client_id) self.assertEqual(task, sample.task) self.assertTrue(previous_absolute_time < sample.absolute_time) previous_absolute_time = sample.absolute_time self.assertTrue(previous_relative_time < sample.relative_time) previous_relative_time = sample.relative_time # we don't have any warmup time period self.assertEqual(metrics.SampleType.Normal, sample.sample_type) # latency equals service time in throughput mode self.assertEqual(sample.latency_ms, sample.service_time_ms) self.assertEqual(1, sample.total_ops) self.assertEqual("docs", sample.total_ops_unit) self.assertEqual(1, sample.request_meta_data["bulk-size"])
def test_infinite_schedule_without_progress_indication(self): task = track.Task("time-based", track.Operation("time-based", track.OperationType.Bulk.name, params={"body": ["a"]}, param_source="driver-test-param-source"), warmup_time_period=0, clients=4, params={"target-throughput": 4, "clients": 4}) schedule_handle = driver.schedule_for(self.test_track, task, 0) schedule = schedule_handle() self.assert_schedule([ (0.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (1.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (2.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (3.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (4.0, metrics.SampleType.Normal, None, {"body": ["a"]}), ], schedule, infinite_schedule=True)
def test_search_task_one_client(self): task = track.Task(track.Operation("search", track.OperationType.Search.name, param_source="driver-test-param-source"), warmup_iterations=3, iterations=5, clients=1, params={"target-throughput": 10, "clients": 1}) schedule = driver.schedule_for(self.test_track, task, 0) expected_schedule = [ (0, metrics.SampleType.Warmup, 1 / 8, {}), (0.1, metrics.SampleType.Warmup, 2 / 8, {}), (0.2, metrics.SampleType.Warmup, 3 / 8, {}), (0.3, metrics.SampleType.Normal, 4 / 8, {}), (0.4, metrics.SampleType.Normal, 5 / 8, {}), (0.5, metrics.SampleType.Normal, 6 / 8, {}), (0.6, metrics.SampleType.Normal, 7 / 8, {}), (0.7, metrics.SampleType.Normal, 8 / 8, {}), ] self.assert_schedule(expected_schedule, schedule)
def test_schedule_with_progress_determined_by_runner(self): task = track.Task("time-based", track.Operation("time-based", "driver-test-runner-with-completion", params={"body": ["a"]}, param_source="driver-test-param-source"), clients=1, params={"target-throughput": 1, "clients": 1}) schedule_handle = driver.schedule_for(self.test_track, task, 0) schedule = schedule_handle() self.assert_schedule([ (0.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (1.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (2.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (3.0, metrics.SampleType.Normal, None, {"body": ["a"]}), (4.0, metrics.SampleType.Normal, None, {"body": ["a"]}), ], schedule, infinite_schedule=True)
def test_execute_schedule_with_progress_determined_by_runner(self, es): es.bulk.return_value = { "errors": False } params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", description="unittest track", indices=None, challenges=None) task = track.Task("time-based", track.Operation("time-based", operation_type="unit-test-recovery", params={ "indices-to-restore": "*", # The runner will determine progress "size": None }, param_source="driver-test-param-source"), warmup_time_period=0, clients=4) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=2, task=task, start_timestamp=time.perf_counter()) cancel = threading.Event() complete = threading.Event() execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete) execute_schedule() samples = sampler.samples self.assertEqual(5, len(samples)) self.assertTrue(self.runner_with_progress.completed) self.assertEqual(1.0, self.runner_with_progress.percent_completed) self.assertFalse(complete.is_set(), "Executor should not auto-complete a normal task") previous_absolute_time = -1.0 previous_relative_time = -1.0 for sample in samples: self.assertEqual(2, sample.client_id) self.assertEqual(task, sample.task) self.assertLess(previous_absolute_time, sample.absolute_time) previous_absolute_time = sample.absolute_time self.assertLess(previous_relative_time, sample.relative_time) previous_relative_time = sample.relative_time # we don't have any warmup time period self.assertEqual(metrics.SampleType.Normal, sample.sample_type) # latency equals service time in throughput mode self.assertEqual(sample.latency_ms, sample.service_time_ms) self.assertEqual(1, sample.total_ops) self.assertEqual("ops", sample.total_ops_unit)
def test_execute_schedule_in_throughput_mode(self, es): es.bulk.return_value = { "errors": False } params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource) test_track = track.Track(name="unittest", short_description="unittest track", description="unittest track", source_root_url="http://example.org", indices=None, challenges=None) task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={ "body": ["action_metadata_line", "index_line"], "action_metadata_present": True }, param_source="driver-test-param-source"), warmup_time_period=0, clients=4, target_throughput=None) schedule = driver.schedule_for(test_track, task, 0) sampler = driver.Sampler(client_id=2, operation=task.operation, start_timestamp=100) driver.execute_schedule(schedule, es, sampler) samples = sampler.samples self.assertTrue(len(samples) > 0) previous_absolute_time = -1.0 previous_relative_time = -1.0 for sample in samples: self.assertEqual(2, sample.client_id) self.assertEqual(task.operation, sample.operation) self.assertTrue(previous_absolute_time < sample.absolute_time) previous_absolute_time = sample.absolute_time self.assertTrue(previous_relative_time < sample.relative_time) previous_relative_time = sample.relative_time # we don't have any warmup time period self.assertEqual(metrics.SampleType.Normal, sample.sample_type) # latency equals service time in throughput mode self.assertEqual(sample.latency_ms, sample.service_time_ms) self.assertEqual(1, sample.total_ops) self.assertEqual("docs", sample.total_ops_unit) self.assertEqual(1, sample.request_meta_data["bulk-size"])
def test_schedule_for_warmup_time_based(self): task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={"body": ["a"], "size": 11}, param_source="driver-test-param-source"), warmup_time_period=0, clients=4, target_throughput=4) invocations = driver.schedule_for(self.test_track, task, 0) self.assert_schedule([ (0.0, metrics.SampleType.Normal, 0, 11, "runner", {"body": ["a"], "size": 11}), (1.0, metrics.SampleType.Normal, 1, 11, "runner", {"body": ["a"], "size": 11}), (2.0, metrics.SampleType.Normal, 2, 11, "runner", {"body": ["a"], "size": 11}), (3.0, metrics.SampleType.Normal, 3, 11, "runner", {"body": ["a"], "size": 11}), (4.0, metrics.SampleType.Normal, 4, 11, "runner", {"body": ["a"], "size": 11}), (5.0, metrics.SampleType.Normal, 5, 11, "runner", {"body": ["a"], "size": 11}), (6.0, metrics.SampleType.Normal, 6, 11, "runner", {"body": ["a"], "size": 11}), (7.0, metrics.SampleType.Normal, 7, 11, "runner", {"body": ["a"], "size": 11}), (8.0, metrics.SampleType.Normal, 8, 11, "runner", {"body": ["a"], "size": 11}), (9.0, metrics.SampleType.Normal, 9, 11, "runner", {"body": ["a"], "size": 11}), (10.0, metrics.SampleType.Normal, 10, 11, "runner", {"body": ["a"], "size": 11}), ], list(invocations))
def test_schedule_for_warmup_time_based(self): task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={"body": ["a"], "size": 11}, param_source="driver-test-param-source"), warmup_time_period=0, clients=4, params={"target-throughput": 4, "clients": 4}) invocations = driver.schedule_for(self.test_track, task, 0) self.assert_schedule([ (0.0, metrics.SampleType.Normal, 1 / 11, {"body": ["a"], "size": 11}), (1.0, metrics.SampleType.Normal, 2 / 11, {"body": ["a"], "size": 11}), (2.0, metrics.SampleType.Normal, 3 / 11, {"body": ["a"], "size": 11}), (3.0, metrics.SampleType.Normal, 4 / 11, {"body": ["a"], "size": 11}), (4.0, metrics.SampleType.Normal, 5 / 11, {"body": ["a"], "size": 11}), (5.0, metrics.SampleType.Normal, 6 / 11, {"body": ["a"], "size": 11}), (6.0, metrics.SampleType.Normal, 7 / 11, {"body": ["a"], "size": 11}), (7.0, metrics.SampleType.Normal, 8 / 11, {"body": ["a"], "size": 11}), (8.0, metrics.SampleType.Normal, 9 / 11, {"body": ["a"], "size": 11}), (9.0, metrics.SampleType.Normal, 10 / 11, {"body": ["a"], "size": 11}), (10.0, metrics.SampleType.Normal, 11 / 11, {"body": ["a"], "size": 11}), ], list(invocations))
async def _run_benchmark(self): # avoid: aiohttp.internal WARNING The object should be created from async function es = self.create_es_clients(sync=False) try: cancel = threading.Event() # allow to buffer more events than by default as we expect to have way more clients. self.sampler = driver.Sampler(start_timestamp=time.perf_counter(), buffer_size=65536) for task in self.challenge.schedule: self.current_tasks = [] aws = [] for sub_task in task: self.current_tasks.append(sub_task) self.logger.info("Running task [%s] with [%d] clients...", sub_task.name, sub_task.clients) for client_id in range(sub_task.clients): schedule = driver.schedule_for(self.track, sub_task, client_id) # used to indicate that we want to prematurely consider this completed. This is *not* due to # cancellation but a regular event in a benchmark and used to model task dependency of parallel tasks. complete = threading.Event() e = driver.AsyncExecutor(client_id, sub_task, schedule, es, self.sampler, cancel, complete, self.abort_on_error) aws.append(e()) # join point _ = await asyncio.gather(*aws) self.logger.info("All clients have finished running task [%s]", task.name) # drain the active samples before we move on to the next task self.update_samples() self.post_process_samples() self.reset_relative_time() self.update_progress_message(task_finished=True) finally: await asyncio.get_event_loop().shutdown_asyncgens() for e in es.values(): await e.transport.close()
def test_schedule_for_time_based(self): task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={"body": ["a"], "size": 11}, param_source="driver-test-param-source"), warmup_time_period=0.1, time_period=0.1, clients=1) invocations = list(driver.schedule_for(self.test_track, task, 0)) self.assertTrue(len(invocations) > 0) last_progress = -1 for invocation_time, sample_type, progress_percent, runner, params in invocations: # we're not throughput throttled self.assertEqual(0, invocation_time) if progress_percent <= 0.5: self.assertEqual(metrics.SampleType.Warmup, sample_type) else: self.assertEqual(metrics.SampleType.Normal, sample_type) self.assertTrue(last_progress < progress_percent) last_progress = progress_percent self.assertTrue(round(progress_percent, 2) >= 0.0, "progress should be >= 0.0 but was [%f]" % progress_percent) self.assertTrue(round(progress_percent, 2) <= 1.0, "progress should be <= 1.0 but was [%f]" % progress_percent) self.assertIsNotNone(runner, "runner must be defined") self.assertEqual({"body": ["a"], "size": 11}, params)