def setUp(self):
params.register_param_source_for_name("driver-test-param-source",
DriverTestParamSource)
params.register_param_source_for_name(
"driver-test-param-source-with-progress",
DriverTestParamSourceWithProgress)
self.test_track = track.Track(name="unittest")
def test_execute_schedule_throughput_throttled(self, es):
es.bulk.return_value = {
"errors": False
}
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
test_track = track.Track(name="unittest", short_description="unittest track", description="unittest track",
source_root_url="http://example.org",
indices=None,
challenges=None)
# in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples
for target_throughput, bounds in {10: [2, 4], 100: [24, 26], 1000: [245, 255]}.items():
task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={
"body": ["action_metadata_line", "index_line"],
"action_metadata_present": True
},
param_source="driver-test-param-source"),
warmup_time_period=0.5, time_period=0.5, clients=4, target_throughput=target_throughput)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=0, operation=task.operation, start_timestamp=0)
driver.execute_schedule(schedule, es, sampler)
samples = sampler.samples
sample_size = len(samples)
lower_bound = bounds[0]
upper_bound = bounds[1]
self.assertTrue(lower_bound <= sample_size <= upper_bound,
msg="Expected sample size to be between %d and %d but was %d" % (lower_bound, upper_bound, sample_size))
def test_cancel_execute_schedule(self, es):
es.bulk.return_value = {
"errors": False
}
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
test_track = track.Track(name="unittest", short_description="unittest track",
source_root_url="http://example.org",
indices=None,
challenges=None)
# in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples
for target_throughput, bounds in {10: [2, 4], 100: [24, 26], 1000: [245, 255]}.items():
task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={
"body": ["action_metadata_line", "index_line"],
"action_metadata_present": True,
"bulk-size": 1
},
param_source="driver-test-param-source"),
warmup_time_period=0.5, time_period=0.5, clients=4,
params={"target-throughput": target_throughput, "clients": 4})
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=0, task=task, start_timestamp=0)
cancel = threading.Event()
complete = threading.Event()
execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete)
cancel.set()
execute_schedule()
samples = sampler.samples
sample_size = len(samples)
self.assertEqual(0, sample_size)
def test_execute_schedule_throughput_throttled(self, es):
es.bulk.return_value = {"errors": False}
params.register_param_source_for_name("driver-test-param-source",
DriverTestParamSource)
test_track = track.Track(name="unittest",
description="unittest track",
indices=None,
challenges=None)
# in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples
for target_throughput, bounds in {
10: [2, 4],
100: [24, 26],
1000: [245, 255]
}.items():
task = track.Task("time-based",
track.Operation(
"time-based",
track.OperationType.Bulk.name,
params={
"body":
["action_metadata_line", "index_line"],
"action_metadata_present":
True,
"bulk-size":
1
},
param_source="driver-test-param-source"),
warmup_time_period=0.5,
time_period=0.5,
clients=4,
params={
"target-throughput": target_throughput,
"clients": 4
},
completes_parent=True)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=0, task=task, start_timestamp=0)
cancel = threading.Event()
complete = threading.Event()
execute_schedule = driver.Executor(task, schedule, es, sampler,
cancel, complete)
execute_schedule()
samples = sampler.samples
sample_size = len(samples)
lower_bound = bounds[0]
upper_bound = bounds[1]
self.assertTrue(
lower_bound <= sample_size <= upper_bound,
msg="Expected sample size to be between %d and %d but was %d" %
(lower_bound, upper_bound, sample_size))
self.assertTrue(
complete.is_set(),
"Executor should auto-complete a task that terminates its parent"
)
def test_can_register_class_as_param_source(self):
source_name = "params-test-class-param-source"
params.register_param_source_for_name(source_name, ParamsRegistrationTests.ParamSourceClass)
source = params.param_source_for_name(source_name, None, {"parameter": 42})
self.assertEqual({"class-key": 42}, source.params())
params._unregister_param_source_for_name(source_name)
def test_can_register_function_as_param_source(self):
source_name = "params-test-function-param-source"
params.register_param_source_for_name(source_name, ParamsRegistrationTests.param_source_function)
source = params.param_source_for_name(source_name, None, {"parameter": 42})
self.assertEqual({"key": 42}, source.params())
params._unregister_param_source_for_name(source_name)
def test_can_register_class_as_param_source(self):
source_name = "params-test-class-param-source"
params.register_param_source_for_name(source_name, ParamsRegistrationTests.ParamSourceClass)
source = params.param_source_for_name(source_name, None, {"parameter": 42})
self.assertEqual({"class-key": 42}, source.params())
params._unregister_param_source_for_name(source_name)
def test_can_register_function_as_param_source(self):
source_name = "params-test-function-param-source"
params.register_param_source_for_name(source_name, ParamsRegistrationTests.param_source_function)
source = params.param_source_for_name(source_name, None, {"parameter": 42})
self.assertEqual({"key": 42}, source.params())
params._unregister_param_source_for_name(source_name)
def setUp(self):
params.register_param_source_for_name("driver-test-param-source",
DriverTestParamSource)
self.test_track = track.Track(name="unittest",
description="unittest track",
source_root_url="http://example.org",
indices=None,
challenges=None)
def test_execute_schedule_in_throughput_mode(self, es):
es.bulk.return_value = {"errors": False}
params.register_param_source_for_name("driver-test-param-source",
DriverTestParamSource)
test_track = track.Track(name="unittest",
description="unittest track",
indices=None,
challenges=None)
task = track.Task(
"time-based",
track.Operation(
"time-based",
track.OperationType.Bulk.name,
params={
"body": ["action_metadata_line", "index_line"],
"action-metadata-present": True,
"bulk-size": 1,
# we need this because DriverTestParamSource does not know that we only have one bulk and hence size() returns incorrect results
"size": 1
},
param_source="driver-test-param-source"),
warmup_time_period=0,
clients=4)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=2,
task=task,
start_timestamp=time.perf_counter())
cancel = threading.Event()
complete = threading.Event()
execute_schedule = driver.Executor(task, schedule, es, sampler, cancel,
complete)
execute_schedule()
samples = sampler.samples
self.assertTrue(len(samples) > 0)
self.assertFalse(complete.is_set(),
"Executor should not auto-complete a normal task")
previous_absolute_time = -1.0
previous_relative_time = -1.0
for sample in samples:
self.assertEqual(2, sample.client_id)
self.assertEqual(task, sample.task)
self.assertLess(previous_absolute_time, sample.absolute_time)
previous_absolute_time = sample.absolute_time
self.assertLess(previous_relative_time, sample.relative_time)
previous_relative_time = sample.relative_time
# we don't have any warmup time period
self.assertEqual(metrics.SampleType.Normal, sample.sample_type)
# latency equals service time in throughput mode
self.assertEqual(sample.latency_ms, sample.service_time_ms)
self.assertEqual(1, sample.total_ops)
self.assertEqual("docs", sample.total_ops_unit)
self.assertEqual(1, sample.request_meta_data["bulk-size"])
def test_execute_schedule_in_throughput_mode(self, es):
es.bulk.return_value = {"errors": False}
params.register_param_source_for_name("driver-test-param-source",
DriverTestParamSource)
test_track = track.Track(name="unittest",
short_description="unittest track",
description="unittest track",
source_root_url="http://example.org",
indices=None,
challenges=None)
task = track.Task(track.Operation(
"time-based",
track.OperationType.Index.name,
params={
"body": ["action_metadata_line", "index_line"],
"action_metadata_present": True
},
param_source="driver-test-param-source"),
warmup_time_period=0,
clients=4,
target_throughput=None)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=2, task=task, start_timestamp=100)
cancel = threading.Event()
driver.execute_schedule(cancel, 0, task.operation, schedule, es,
sampler)
samples = sampler.samples
self.assertTrue(len(samples) > 0)
previous_absolute_time = -1.0
previous_relative_time = -1.0
for sample in samples:
self.assertEqual(2, sample.client_id)
self.assertEqual(task, sample.task)
self.assertTrue(previous_absolute_time < sample.absolute_time)
previous_absolute_time = sample.absolute_time
self.assertTrue(previous_relative_time < sample.relative_time)
previous_relative_time = sample.relative_time
# we don't have any warmup time period
self.assertEqual(metrics.SampleType.Normal, sample.sample_type)
# latency equals service time in throughput mode
self.assertEqual(sample.latency_ms, sample.service_time_ms)
self.assertEqual(1, sample.total_ops)
self.assertEqual("docs", sample.total_ops_unit)
self.assertEqual(1, sample.request_meta_data["bulk-size"])
def test_execute_schedule_in_throughput_mode(self, es):
es.bulk.return_value = {
"errors": False
}
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
test_track = track.Track(name="unittest", short_description="unittest track",
source_root_url="http://example.org",
indices=None,
challenges=None)
task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={
"body": ["action_metadata_line", "index_line"],
"action_metadata_present": True,
"bulk-size": 1,
# we need this because DriverTestParamSource does not know that we only have one bulk and hence size() returns incorrect results
"size": 1
},
param_source="driver-test-param-source"),
warmup_time_period=0, clients=4)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=2, task=task, start_timestamp=100)
cancel = threading.Event()
complete = threading.Event()
execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete)
execute_schedule()
samples = sampler.samples
self.assertTrue(len(samples) > 0)
self.assertFalse(complete.is_set(), "Executor should not auto-complete a normal task")
previous_absolute_time = -1.0
previous_relative_time = -1.0
for sample in samples:
self.assertEqual(2, sample.client_id)
self.assertEqual(task, sample.task)
self.assertTrue(previous_absolute_time < sample.absolute_time)
previous_absolute_time = sample.absolute_time
self.assertTrue(previous_relative_time < sample.relative_time)
previous_relative_time = sample.relative_time
# we don't have any warmup time period
self.assertEqual(metrics.SampleType.Normal, sample.sample_type)
# latency equals service time in throughput mode
self.assertEqual(sample.latency_ms, sample.service_time_ms)
self.assertEqual(1, sample.total_ops)
self.assertEqual("docs", sample.total_ops_unit)
self.assertEqual(1, sample.request_meta_data["bulk-size"])
def test_execute_schedule_with_progress_determined_by_runner(self, es):
es.bulk.return_value = {
"errors": False
}
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
test_track = track.Track(name="unittest", description="unittest track",
indices=None,
challenges=None)
task = track.Task("time-based", track.Operation("time-based", operation_type="unit-test-recovery", params={
"indices-to-restore": "*",
# The runner will determine progress
"size": None
}, param_source="driver-test-param-source"), warmup_time_period=0, clients=4)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=2, task=task, start_timestamp=time.perf_counter())
cancel = threading.Event()
complete = threading.Event()
execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete)
execute_schedule()
samples = sampler.samples
self.assertEqual(5, len(samples))
self.assertTrue(self.runner_with_progress.completed)
self.assertEqual(1.0, self.runner_with_progress.percent_completed)
self.assertFalse(complete.is_set(), "Executor should not auto-complete a normal task")
previous_absolute_time = -1.0
previous_relative_time = -1.0
for sample in samples:
self.assertEqual(2, sample.client_id)
self.assertEqual(task, sample.task)
self.assertLess(previous_absolute_time, sample.absolute_time)
previous_absolute_time = sample.absolute_time
self.assertLess(previous_relative_time, sample.relative_time)
previous_relative_time = sample.relative_time
# we don't have any warmup time period
self.assertEqual(metrics.SampleType.Normal, sample.sample_type)
# latency equals service time in throughput mode
self.assertEqual(sample.latency_ms, sample.service_time_ms)
self.assertEqual(1, sample.total_ops)
self.assertEqual("ops", sample.total_ops_unit)
def test_execute_schedule_in_throughput_mode(self, es):
es.bulk.return_value = {
"errors": False
}
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
test_track = track.Track(name="unittest", short_description="unittest track", description="unittest track",
source_root_url="http://example.org",
indices=None,
challenges=None)
task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={
"body": ["action_metadata_line", "index_line"],
"action_metadata_present": True
},
param_source="driver-test-param-source"),
warmup_time_period=0, clients=4, target_throughput=None)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=2, operation=task.operation, start_timestamp=100)
driver.execute_schedule(schedule, es, sampler)
samples = sampler.samples
self.assertTrue(len(samples) > 0)
previous_absolute_time = -1.0
previous_relative_time = -1.0
for sample in samples:
self.assertEqual(2, sample.client_id)
self.assertEqual(task.operation, sample.operation)
self.assertTrue(previous_absolute_time < sample.absolute_time)
previous_absolute_time = sample.absolute_time
self.assertTrue(previous_relative_time < sample.relative_time)
previous_relative_time = sample.relative_time
# we don't have any warmup time period
self.assertEqual(metrics.SampleType.Normal, sample.sample_type)
# latency equals service time in throughput mode
self.assertEqual(sample.latency_ms, sample.service_time_ms)
self.assertEqual(1, sample.total_ops)
self.assertEqual("docs", sample.total_ops_unit)
self.assertEqual(1, sample.request_meta_data["bulk-size"])
def setUp(self):
self.test_track = track.Track(name="unittest")
self.runner_with_progress = SchedulerTests.RunnerWithProgress()
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
runner.register_default_runners()
runner.register_runner("driver-test-runner-with-completion", self.runner_with_progress)
def test_run_benchmark(self):
cfg = config.Config()
cfg.add(config.Scope.application, "system", "env.name", "unittest")
cfg.add(
config.Scope.application, "system", "time.start",
datetime(year=2017, month=8, day=20, hour=1, minute=0, second=0))
cfg.add(config.Scope.application, "system", "race.id",
"6ebc6e53-ee20-4b0c-99b4-09697987e9f4")
cfg.add(config.Scope.application, "system", "offline.mode", False)
cfg.add(config.Scope.application, "driver", "on.error", "abort")
cfg.add(config.Scope.application, "driver", "profiling", False)
cfg.add(config.Scope.application, "reporting", "datastore.type",
"in-memory")
cfg.add(config.Scope.application, "track", "params", {})
cfg.add(config.Scope.application, "track", "test.mode.enabled", True)
cfg.add(config.Scope.application, "telemetry", "devices", [])
cfg.add(config.Scope.application, "telemetry", "params", {})
cfg.add(config.Scope.application, "mechanic", "car.names",
["external"])
cfg.add(config.Scope.application, "mechanic", "skip.rest.api.check",
True)
cfg.add(
config.Scope.application, "client", "hosts",
AsyncDriverTests.Holder(all_hosts={"default": ["localhost:9200"]}))
cfg.add(config.Scope.application, "client", "options",
AsyncDriverTests.Holder(all_client_options={"default": {}}))
params.register_param_source_for_name("bulk-param-source",
AsyncDriverTestParamSource)
task = track.Task(
name="bulk-index",
operation=track.Operation(
"bulk-index",
track.OperationType.Bulk.name,
params={
"body": ["action_metadata_line", "index_line"],
"action-metadata-present": True,
"bulk-size": 1,
# we need this because the parameter source does not know that we only have one
# bulk and hence size() returns incorrect results
"size": 1
},
param_source="bulk-param-source"),
warmup_iterations=0,
iterations=1,
clients=1)
current_challenge = track.Challenge(name="default",
default=True,
schedule=[task])
current_track = track.Track(name="unit-test",
challenges=[current_challenge])
driver = async_driver.AsyncDriver(
cfg,
current_track,
current_challenge,
es_client_factory_class=StaticClientFactory)
distribution_flavor, distribution_version, revision = driver.setup()
self.assertEqual("oss", distribution_flavor)
self.assertEqual("7.3.0", distribution_version)
self.assertEqual("de777fa", revision)
metrics_store_representation = driver.run()
metric_store = metrics.metrics_store(cfg,
read_only=True,
track=current_track,
challenge=current_challenge)
metric_store.bulk_add(metrics_store_representation)
self.assertIsNotNone(
metric_store.get_one(name="latency",
task="bulk-index",
sample_type=metrics.SampleType.Normal))
self.assertIsNotNone(
metric_store.get_one(name="service_time",
task="bulk-index",
sample_type=metrics.SampleType.Normal))
self.assertIsNotNone(
metric_store.get_one(name="processing_time",
task="bulk-index",
sample_type=metrics.SampleType.Normal))
self.assertIsNotNone(
metric_store.get_one(name="throughput",
task="bulk-index",
sample_type=metrics.SampleType.Normal))
self.assertIsNotNone(
metric_store.get_one(name="node_total_young_gen_gc_time",
sample_type=metrics.SampleType.Normal))
self.assertIsNotNone(
metric_store.get_one(name="node_total_old_gen_gc_time",
sample_type=metrics.SampleType.Normal))
# ensure that there are not more documents than we expect
self.assertEqual(6,
len(metric_store.docs),
msg=json.dumps(metric_store.docs, indent=2))
def setUp(self):
params.register_param_source_for_name("driver-test-param-source",
DriverTestParamSource)
def register_param_source(self, name, param_source):
params.register_param_source_for_name(name, param_source)
def setUp(self):
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
runner.register_default_runners()
self.test_track = track.Track(name="unittest")
def register_param_source(self, name, param_source):
params.register_param_source_for_name(name, param_source)
def setUp(self):
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
self.test_track = track.Track(name="unittest", short_description="unittest track",
source_root_url="http://example.org",
indices=None,
challenges=None)
def setUp(self):
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
