def test_allocates_mixed_tasks(self):
op1 = track.Operation("index",
track.OperationType.Index,
param_source="driver-test-param-source")
op2 = track.Operation("stats",
track.OperationType.IndicesStats,
param_source="driver-test-param-source")
op3 = track.Operation("search",
track.OperationType.Search,
param_source="driver-test-param-source")
index = track.Task(op1)
stats = track.Task(op2)
search = track.Task(op3)
allocator = driver.Allocator([
index,
track.Parallel([index, stats, stats]), index, index,
track.Parallel([search, search, search])
])
self.assertEqual(3, allocator.clients)
# 1 join point, 1 op, 1 jp, 1 (parallel) op, 1 jp, 1 op, 1 jp, 1 op, 1 jp, 1 (parallel) op, 1 jp
self.assertEqual(11, len(allocator.allocations[0]))
self.assertEqual(11, len(allocator.allocations[1]))
self.assertEqual(11, len(allocator.allocations[2]))
self.assertEqual(6, len(allocator.join_points))
self.assertEqual([{op1}, {op1, op2}, {op1}, {op1}, {op3}],
allocator.operations_per_joinpoint)
def test_allocates_more_tasks_than_clients(self):
op1 = track.Operation("index-a", track.OperationType.Index, param_source="driver-test-param-source")
op2 = track.Operation("index-b", track.OperationType.Index, param_source="driver-test-param-source")
op3 = track.Operation("index-c", track.OperationType.Index, param_source="driver-test-param-source")
op4 = track.Operation("index-d", track.OperationType.Index, param_source="driver-test-param-source")
op5 = track.Operation("index-e", track.OperationType.Index, param_source="driver-test-param-source")
index_a = track.Task(op1)
index_b = track.Task(op2, completes_parent=True)
index_c = track.Task(op3)
index_d = track.Task(op4)
index_e = track.Task(op5)
allocator = driver.Allocator([track.Parallel(tasks=[index_a, index_b, index_c, index_d, index_e], clients=2)])
self.assertEqual(2, allocator.clients)
allocations = allocator.allocations
# 2 clients
self.assertEqual(2, len(allocations))
# join_point, index_a, index_c, index_e, join_point
self.assertEqual(5, len(allocations[0]))
# we really have no chance to extract the join point so we just take what is there...
self.assertEqual([allocations[0][0], index_a, index_c, index_e, allocations[0][4]], allocations[0])
# join_point, index_a, index_c, None, join_point
self.assertEqual(5, len(allocator.allocations[1]))
self.assertEqual([allocations[1][0], index_b, index_d, None, allocations[1][4]], allocations[1])
self.assertEqual([{op1, op2, op3, op4, op5}], allocator.operations_per_joinpoint)
self.assertEqual(2, len(allocator.join_points))
final_join_point = allocator.join_points[1]
self.assertTrue(final_join_point.preceding_task_completes_parent)
self.assertEqual(1, final_join_point.num_clients_executing_completing_task)
self.assertEqual([1], final_join_point.clients_executing_completing_task)
def test_considers_number_of_clients_per_subtask(self):
op1 = track.Operation("index-a", track.OperationType.Index, param_source="driver-test-param-source")
op2 = track.Operation("index-b", track.OperationType.Index, param_source="driver-test-param-source")
op3 = track.Operation("index-c", track.OperationType.Index, param_source="driver-test-param-source")
index_a = track.Task(op1)
index_b = track.Task(op2)
index_c = track.Task(op3, clients=2)
allocator = driver.Allocator([track.Parallel(tasks=[index_a, index_b, index_c], clients=3)])
self.assertEqual(3, allocator.clients)
allocations = allocator.allocations
self.assertEqual(3, len(allocations))
# join_point, index_a, index_c, join_point
self.assertEqual(4, len(allocations[0]))
# we really have no chance to extract the join point so we just take what is there...
self.assertEqual([allocations[0][0], index_a, index_c, allocations[0][3]], allocations[0])
# join_point, index_b, None, join_point
self.assertEqual(4, len(allocator.allocations[1]))
self.assertEqual([allocations[1][0], index_b, None, allocations[1][3]], allocations[1])
self.assertEqual(4, len(allocator.allocations[2]))
self.assertEqual([allocations[2][0], index_c, None, allocations[2][3]], allocations[2])
self.assertEqual([{op1, op2, op3}], allocator.operations_per_joinpoint)
def test_considers_number_of_clients_per_subtask(self):
op1 = track.Operation("index-a",
track.OperationType.Index,
param_source="driver-test-param-source")
op2 = track.Operation("index-b",
track.OperationType.Index,
param_source="driver-test-param-source")
op3 = track.Operation("index-c",
track.OperationType.Index,
param_source="driver-test-param-source")
index_a = track.Task(op1)
index_b = track.Task(op2)
index_c = track.Task(op3, clients=2, completes_parent=True)
allocator = driver.Allocator(
[track.Parallel(tasks=[index_a, index_b, index_c], clients=3)])
self.assertEqual(3, allocator.clients)
allocations = allocator.allocations
# 3 clients
self.assertEqual(3, len(allocations))
# join_point, index_a, index_c, join_point
self.assertEqual(4, len(allocations[0]))
# we really have no chance to extract the join point so we just take what is there...
self.assertEqual(
[allocations[0][0], index_a, index_c, allocations[0][3]],
allocations[0])
# join_point, index_b, None, join_point
self.assertEqual(4, len(allocator.allocations[1]))
self.assertEqual([allocations[1][0], index_b, None, allocations[1][3]],
allocations[1])
self.assertEqual(4, len(allocator.allocations[2]))
self.assertEqual([allocations[2][0], index_c, None, allocations[2][3]],
allocations[2])
self.assertEqual([{op1, op2, op3}], allocator.operations_per_joinpoint)
self.assertEqual(2, len(allocator.join_points))
final_join_point = allocator.join_points[1]
self.assertTrue(final_join_point.preceding_task_completes_parent)
# task index_c has two clients, hence we have to wait for two clients to finish
self.assertEqual(
2, final_join_point.num_clients_executing_completing_task)
self.assertEqual([2, 0],
final_join_point.clients_executing_completing_task)
def test_cancel_execute_schedule(self, es):
es.bulk.return_value = {
"errors": False
}
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
test_track = track.Track(name="unittest", short_description="unittest track",
source_root_url="http://example.org",
indices=None,
challenges=None)
# in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples
for target_throughput, bounds in {10: [2, 4], 100: [24, 26], 1000: [245, 255]}.items():
task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={
"body": ["action_metadata_line", "index_line"],
"action_metadata_present": True,
"bulk-size": 1
},
param_source="driver-test-param-source"),
warmup_time_period=0.5, time_period=0.5, clients=4,
params={"target-throughput": target_throughput, "clients": 4})
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=0, task=task, start_timestamp=0)
cancel = threading.Event()
complete = threading.Event()
execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete)
cancel.set()
execute_schedule()
samples = sampler.samples
sample_size = len(samples)
self.assertEqual(0, sample_size)
def test_eternal_schedule(self):
task = track.Task("time-based",
track.Operation(
"time-based",
track.OperationType.Bulk.name,
params={"body": ["a"]},
param_source="driver-test-param-source"),
warmup_time_period=0,
clients=4,
params={
"target-throughput": 4,
"clients": 4
})
invocations = driver.schedule_for(self.test_track, task, 0)
self.assert_schedule([
(0.0, metrics.SampleType.Normal, None, {
"body": ["a"]
}),
(1.0, metrics.SampleType.Normal, None, {
"body": ["a"]
}),
(2.0, metrics.SampleType.Normal, None, {
"body": ["a"]
}),
(3.0, metrics.SampleType.Normal, None, {
"body": ["a"]
}),
(4.0, metrics.SampleType.Normal, None, {
"body": ["a"]
}),
],
invocations,
eternal_schedule=True)
def test_execute_schedule_throughput_throttled(self, es):
es.bulk.return_value = {"errors": False}
params.register_param_source_for_name("driver-test-param-source",
DriverTestParamSource)
test_track = track.Track(name="unittest",
description="unittest track",
indices=None,
challenges=None)
# in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples
for target_throughput, bounds in {
10: [2, 4],
100: [24, 26],
1000: [245, 255]
}.items():
task = track.Task("time-based",
track.Operation(
"time-based",
track.OperationType.Bulk.name,
params={
"body":
["action_metadata_line", "index_line"],
"action_metadata_present":
True,
"bulk-size":
1
},
param_source="driver-test-param-source"),
warmup_time_period=0.5,
time_period=0.5,
clients=4,
params={
"target-throughput": target_throughput,
"clients": 4
},
completes_parent=True)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=0, task=task, start_timestamp=0)
cancel = threading.Event()
complete = threading.Event()
execute_schedule = driver.Executor(task, schedule, es, sampler,
cancel, complete)
execute_schedule()
samples = sampler.samples
sample_size = len(samples)
lower_bound = bounds[0]
upper_bound = bounds[1]
self.assertTrue(
lower_bound <= sample_size <= upper_bound,
msg="Expected sample size to be between %d and %d but was %d" %
(lower_bound, upper_bound, sample_size))
self.assertTrue(
complete.is_set(),
"Executor should auto-complete a task that terminates its parent"
)
def test_schedule_param_source_determines_iterations_no_warmup(self):
# we neither define any time-period nor any iteration count on the task.
task = track.Task("bulk-index",
track.Operation(
"bulk-index",
track.OperationType.Bulk.name,
params={
"body": ["a"],
"size": 3
},
param_source="driver-test-param-source"),
clients=1,
params={
"target-throughput": 4,
"clients": 4
})
invocations = driver.schedule_for(self.test_track, task, 0)
self.assert_schedule([
(0.0, metrics.SampleType.Normal, 1 / 3, {
"body": ["a"],
"size": 3
}),
(1.0, metrics.SampleType.Normal, 2 / 3, {
"body": ["a"],
"size": 3
}),
(2.0, metrics.SampleType.Normal, 3 / 3, {
"body": ["a"],
"size": 3
}),
], list(invocations))
def test_execute_schedule_aborts_on_error(self, es):
class ExpectedUnitTestException(Exception):
pass
def run(*args, **kwargs):
raise ExpectedUnitTestException()
task = track.Task("no-op",
track.Operation(
"no-op",
track.OperationType.Bulk.name,
params={},
param_source="driver-test-param-source"),
warmup_time_period=0.5,
time_period=0.5,
clients=4,
params={"clients": 4})
schedule = [(0, metrics.SampleType.Warmup, 0,
self.context_managed(run), None)]
sampler = driver.Sampler(client_id=0, task=None, start_timestamp=0)
cancel = threading.Event()
complete = threading.Event()
execute_schedule = driver.Executor(task, schedule, es, sampler, cancel,
complete)
with self.assertRaises(ExpectedUnitTestException):
execute_schedule()
es.assert_not_called()
def test_calculate_simple_index_stats(self):
cfg = config.Config()
cfg.add(config.Scope.application, "system", "env.name", "unittest")
store = metrics.InMemoryMetricsStore(config=cfg, clear=True)
store.open(datetime.datetime.now(), "test", "unittest", "unittest_car")
store.put_value_cluster_level("throughput", 500, unit="docs/s", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("throughput", 1000, unit="docs/s", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("throughput", 2000, unit="docs/s", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("latency", 2800, unit="ms", operation="index", operation_type=track.OperationType.Index,
sample_type=metrics.SampleType.Warmup)
store.put_value_cluster_level("latency", 200, unit="ms", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("latency", 220, unit="ms", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("latency", 225, unit="ms", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("service_time", 250, unit="ms", operation="index", operation_type=track.OperationType.Index,
sample_type=metrics.SampleType.Warmup)
store.put_value_cluster_level("service_time", 190, unit="ms", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("service_time", 200, unit="ms", operation="index", operation_type=track.OperationType.Index)
store.put_value_cluster_level("service_time", 215, unit="ms", operation="index", operation_type=track.OperationType.Index)
index = track.Task(operation=track.Operation(name="index", operation_type=track.OperationType.Index, granularity_unit="docs/s"))
challenge = track.Challenge(name="unittest", description="", index_settings=None, schedule=[index])
stats = reporter.Stats(store, challenge)
self.assertEqual((500, 1000, 2000, "docs/s"), stats.op_metrics["index"]["throughput"])
self.assertEqual(collections.OrderedDict([(50.0, 220), (100, 225)]), stats.op_metrics["index"]["latency"])
self.assertEqual(collections.OrderedDict([(50.0, 200), (100, 215)]), stats.op_metrics["index"]["service_time"])
def test_single_metrics_aggregation(self):
op = track.Operation("index", track.OperationType.Index, param_source="driver-test-param-source")
samples = [
driver.Sample(0, 1470838595, 21, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 1, 1 / 9),
driver.Sample(0, 1470838596, 22, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 2, 2 / 9),
driver.Sample(0, 1470838597, 23, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 3, 3 / 9),
driver.Sample(0, 1470838598, 24, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 4, 4 / 9),
driver.Sample(0, 1470838599, 25, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 5, 5 / 9),
driver.Sample(0, 1470838600, 26, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 6, 6 / 9),
driver.Sample(1, 1470838598.5, 24.5, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 4.5, 7 / 9),
driver.Sample(1, 1470838599.5, 25.5, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 5.5, 8 / 9),
driver.Sample(1, 1470838600.5, 26.5, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 6.5, 9 / 9)
]
aggregated = driver.calculate_global_throughput(samples)
self.assertIn(op, aggregated)
self.assertEqual(1, len(aggregated))
throughput = aggregated[op]
self.assertEqual(6, len(throughput))
self.assertEqual((1470838595, 21, metrics.SampleType.Normal, 5000, "docs/s"), throughput[0])
self.assertEqual((1470838596, 22, metrics.SampleType.Normal, 5000, "docs/s"), throughput[1])
self.assertEqual((1470838597, 23, metrics.SampleType.Normal, 5000, "docs/s"), throughput[2])
self.assertEqual((1470838598, 24, metrics.SampleType.Normal, 5000, "docs/s"), throughput[3])
self.assertEqual((1470838599, 25, metrics.SampleType.Normal, 6000, "docs/s"), throughput[4])
self.assertEqual((1470838600, 26, metrics.SampleType.Normal, 6666.666666666667, "docs/s"), throughput[5])
def test_search_task_one_client(self):
task = track.Task("search",
track.Operation(
"search",
track.OperationType.Search.name,
param_source="driver-test-param-source"),
warmup_iterations=3,
iterations=5,
clients=1,
params={
"target-throughput": 10,
"clients": 1
})
schedule = driver.schedule_for(self.test_track, task, 0)
expected_schedule = [
(0, metrics.SampleType.Warmup, 1 / 8, {}),
(0.1, metrics.SampleType.Warmup, 2 / 8, {}),
(0.2, metrics.SampleType.Warmup, 3 / 8, {}),
(0.3, metrics.SampleType.Normal, 4 / 8, {}),
(0.4, metrics.SampleType.Normal, 5 / 8, {}),
(0.5, metrics.SampleType.Normal, 6 / 8, {}),
(0.6, metrics.SampleType.Normal, 7 / 8, {}),
(0.7, metrics.SampleType.Normal, 8 / 8, {}),
]
self.assert_schedule(expected_schedule, schedule)
def test_schedule_for_time_based(self):
task = track.Task("time-based", track.Operation("time-based", track.OperationType.Bulk.name, params={"body": ["a"], "size": 11},
param_source="driver-test-param-source"), warmup_time_period=0.1, time_period=0.1,
clients=1)
schedule_handle = driver.schedule_for(self.test_track, task, 0)
schedule = list(schedule_handle())
self.assertTrue(len(schedule) > 0)
last_progress = -1
for invocation_time, sample_type, progress_percent, runner, params in schedule:
# we're not throughput throttled
self.assertEqual(0, invocation_time)
if progress_percent <= 0.5:
self.assertEqual(metrics.SampleType.Warmup, sample_type)
else:
self.assertEqual(metrics.SampleType.Normal, sample_type)
self.assertTrue(last_progress < progress_percent)
last_progress = progress_percent
self.assertTrue(round(progress_percent, 2) >= 0.0, "progress should be >= 0.0 but was [%f]" % progress_percent)
self.assertTrue(round(progress_percent, 2) <= 1.0, "progress should be <= 1.0 but was [%f]" % progress_percent)
self.assertIsNotNone(runner, "runner must be defined")
self.assertEqual({"body": ["a"], "size": 11}, params)
def test_execute_schedule_throughput_throttled(self, es):
es.bulk.return_value = {
"errors": False
}
params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
test_track = track.Track(name="unittest", short_description="unittest track", description="unittest track",
source_root_url="http://example.org",
indices=None,
challenges=None)
# in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples
for target_throughput, bounds in {10: [2, 4], 100: [24, 26], 1000: [245, 255]}.items():
task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={
"body": ["action_metadata_line", "index_line"],
"action_metadata_present": True
},
param_source="driver-test-param-source"),
warmup_time_period=0.5, time_period=0.5, clients=4, target_throughput=target_throughput)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=0, operation=task.operation, start_timestamp=0)
driver.execute_schedule(schedule, es, sampler)
samples = sampler.samples
sample_size = len(samples)
lower_bound = bounds[0]
upper_bound = bounds[1]
self.assertTrue(lower_bound <= sample_size <= upper_bound,
msg="Expected sample size to be between %d and %d but was %d" % (lower_bound, upper_bound, sample_size))
def test_store_race(self):
self.cfg.add(config.Scope.application, "system", "pipeline", "unittest-pipeline")
self.cfg.add(config.Scope.application, "system", "user.tag", "")
self.cfg.add(config.Scope.application, "benchmarks", "challenge", "index-and-search")
self.cfg.add(config.Scope.application, "benchmarks", "car", "defaults")
self.cfg.add(config.Scope.application, "benchmarks", "laps", 1)
self.cfg.add(config.Scope.application, "launcher", "external.target.hosts", [{"host": "localhost", "port": "9200"}])
self.cfg.add(config.Scope.application, "source", "revision", "latest")
self.cfg.add(config.Scope.application, "source", "distribution.version", "5.0.0")
index = "tests"
type = "test-type"
schedule = [
track.Task(track.Operation("index", track.OperationType.Index, None)),
track.Task(track.Operation("search-all", track.OperationType.Search, None)),
]
t = track.Track(name="unittest", short_description="unittest track", description="unittest track",
source_root_url="http://example.org",
indices=[track.Index(name=index, types=[track.Type(name=type, mapping_file=None)])],
challenges=[
track.Challenge(name="index-and-search", description="Index & Search", index_settings=None, schedule=schedule)
])
self.race_store.store_race(t)
expected_doc = {
"environment": "unittest-env",
"trial-timestamp": "20160131T000000Z",
"pipeline": "unittest-pipeline",
"revision": "latest",
"distribution-version": "5.0.0",
"track": "unittest",
"laps": 1,
"selected-challenge": {
"name": "index-and-search",
"operations": [
"index",
"search-all"
]
},
"car": "defaults",
"target-hosts": ["localhost:9200"],
"user-tag": ""
}
self.es_mock.index.assert_called_with(index="rally-2016", doc_type="races", item=expected_doc)
def test_a_task_completes_the_parallel_structure(self):
opA = track.Operation("index-completing", track.OperationType.Index, param_source="driver-test-param-source")
opB = track.Operation("index-non-completing", track.OperationType.Index, param_source="driver-test-param-source")
taskA = track.Task(opA, completes_parent=True)
taskB = track.Task(opB)
allocator = driver.Allocator([track.Parallel([taskA, taskB])])
self.assertEqual(2, allocator.clients)
self.assertEqual(3, len(allocator.allocations[0]))
self.assertEqual(3, len(allocator.allocations[1]))
self.assertEqual(2, len(allocator.join_points))
self.assertEqual([{opA, opB}], allocator.operations_per_joinpoint)
final_join_point = allocator.join_points[1]
self.assertTrue(final_join_point.preceding_task_completes_parent)
self.assertEqual(1, final_join_point.num_clients_executing_completing_task)
self.assertEqual([0], final_join_point.clients_executing_completing_task)
def test_execute_schedule_in_throughput_mode(self, es):
es.bulk.return_value = {"errors": False}
params.register_param_source_for_name("driver-test-param-source",
DriverTestParamSource)
test_track = track.Track(name="unittest",
description="unittest track",
indices=None,
challenges=None)
task = track.Task(
"time-based",
track.Operation(
"time-based",
track.OperationType.Bulk.name,
params={
"body": ["action_metadata_line", "index_line"],
"action-metadata-present": True,
"bulk-size": 1,
# we need this because DriverTestParamSource does not know that we only have one bulk and hence size() returns incorrect results
"size": 1
},
param_source="driver-test-param-source"),
warmup_time_period=0,
clients=4)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=2,
task=task,
start_timestamp=time.perf_counter())
cancel = threading.Event()
complete = threading.Event()
execute_schedule = driver.Executor(task, schedule, es, sampler, cancel,
complete)
execute_schedule()
samples = sampler.samples
self.assertTrue(len(samples) > 0)
self.assertFalse(complete.is_set(),
"Executor should not auto-complete a normal task")
previous_absolute_time = -1.0
previous_relative_time = -1.0
for sample in samples:
self.assertEqual(2, sample.client_id)
self.assertEqual(task, sample.task)
self.assertLess(previous_absolute_time, sample.absolute_time)
previous_absolute_time = sample.absolute_time
self.assertLess(previous_relative_time, sample.relative_time)
previous_relative_time = sample.relative_time
# we don't have any warmup time period
self.assertEqual(metrics.SampleType.Normal, sample.sample_type)
# latency equals service time in throughput mode
self.assertEqual(sample.latency_ms, sample.service_time_ms)
self.assertEqual(1, sample.total_ops)
self.assertEqual("docs", sample.total_ops_unit)
self.assertEqual(1, sample.request_meta_data["bulk-size"])
def test_store_race(self):
from esrally import time
schedule = [
track.Task("index #1",
track.Operation("index", track.OperationType.Bulk))
]
t = track.Track(
name="unittest",
indices=[track.Index(name="tests", types=["test-type"])],
challenges=[
track.Challenge(name="index", default=True, schedule=schedule)
])
race = metrics.Race(rally_version="0.4.4",
environment_name="unittest",
trial_id=FileRaceStoreTests.TRIAL_ID,
trial_timestamp=FileRaceStoreTests.TRIAL_TIMESTAMP,
pipeline="from-sources",
user_tags={"os": "Linux"},
track=t,
track_params={"clients": 12},
challenge=t.default_challenge,
car="4gheap",
total_laps=12,
cluster=FileRaceStoreTests.DictHolder({
"distribution-version":
"5.0.0",
"nodes": [{
"node_name": "node0",
"ip": "127.0.0.1"
}]
}),
lap_results=[],
results=FileRaceStoreTests.DictHolder({
"young_gc_time":
100,
"old_gc_time":
5,
"op_metrics": [{
"task": "index #1",
"operation": "index",
"throughput": {
"min": 1000,
"median": 1250,
"max": 1500,
"unit": "docs/s"
}
}]
}))
self.race_store.store_race(race)
retrieved_race = self.race_store.find_by_timestamp(
timestamp=time.to_iso8601(FileRaceStoreTests.TRIAL_TIMESTAMP))
self.assertEqual(race.trial_timestamp, retrieved_race.trial_timestamp)
self.assertEqual(1, len(self.race_store.list()))
def test_allocates_one_task(self):
op = track.Operation("index", track.OperationType.Index, param_source="driver-test-param-source")
task = track.Task(op)
allocator = driver.Allocator([task])
self.assertEqual(1, allocator.clients)
self.assertEqual(3, len(allocator.allocations[0]))
self.assertEqual(2, len(allocator.join_points))
self.assertEqual([{op}], allocator.operations_per_joinpoint)
def test_allocates_two_parallel_tasks(self):
op = track.Operation("index", track.OperationType.Index)
task = track.Task(op)
allocator = driver.Allocator([track.Parallel([task, task])])
self.assertEqual(2, allocator.clients)
self.assertEqual(3, len(allocator.allocations[0]))
self.assertEqual(3, len(allocator.allocations[1]))
self.assertEqual(2, len(allocator.join_points))
self.assertEqual([{op}], allocator.operations_per_joinpoint)
def test_schedule_defaults_to_iteration_based(self):
# no time-period and no iterations specified on the task. Also, the parameter source does not define a size.
task = track.Task("bulk-index", track.Operation("bulk-index", track.OperationType.Bulk.name, params={"body": ["a"]},
param_source="driver-test-param-source"),
clients=1, params={"target-throughput": 4, "clients": 4})
invocations = driver.schedule_for(self.test_track, task, 0)
self.assert_schedule([
(0.0, metrics.SampleType.Normal, 1 / 1, {"body": ["a"]}),
], list(invocations))
def test_allocates_two_serial_tasks(self):
op = track.Operation("index", track.OperationType.Index, param_source="driver-test-param-source")
task = track.Task(op)
allocator = driver.Allocator([task, task])
self.assertEqual(1, allocator.clients)
# we have two operations and three join points
self.assertEqual(5, len(allocator.allocations[0]))
self.assertEqual(3, len(allocator.join_points))
self.assertEqual([{op}, {op}], allocator.operations_per_joinpoint)
def test_calculate_simple_index_stats(self):
cfg = config.Config()
cfg.add(config.Scope.application, "system", "env.name", "unittest")
cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.now())
cfg.add(config.Scope.application, "system", "trial.id", "6ebc6e53-ee20-4b0c-99b4-09697987e9f4")
cfg.add(config.Scope.application, "reporting", "datastore.type", "in-memory")
cfg.add(config.Scope.application, "mechanic", "car.names", ["unittest_car"])
cfg.add(config.Scope.application, "mechanic", "car.params", {})
cfg.add(config.Scope.application, "mechanic", "plugin.params", {})
cfg.add(config.Scope.application, "race", "laps", 1)
cfg.add(config.Scope.application, "race", "user.tag", "")
cfg.add(config.Scope.application, "race", "pipeline", "from-sources-skip-build")
cfg.add(config.Scope.application, "track", "params", {})
index = track.Task(name="index #1", operation=track.Operation(name="index", operation_type=track.OperationType.Bulk, params=None))
challenge = track.Challenge(name="unittest", schedule=[index], default=True)
t = track.Track("unittest", "unittest-track", challenges=[challenge])
store = metrics.metrics_store(cfg, read_only=False, track=t, challenge=challenge)
store.lap = 1
store.put_value_cluster_level("throughput", 500, unit="docs/s", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("throughput", 1000, unit="docs/s", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("throughput", 2000, unit="docs/s", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("latency", 2800, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
sample_type=metrics.SampleType.Warmup)
store.put_value_cluster_level("latency", 200, unit="ms", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("latency", 220, unit="ms", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("latency", 225, unit="ms", task="index #1", operation_type=track.OperationType.Bulk)
store.put_value_cluster_level("service_time", 250, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
sample_type=metrics.SampleType.Warmup, meta_data={"success": False})
store.put_value_cluster_level("service_time", 190, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
meta_data={"success": True})
store.put_value_cluster_level("service_time", 200, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
meta_data={"success": False})
store.put_value_cluster_level("service_time", 215, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
meta_data={"success": True})
store.put_count_node_level("rally-node-0", "final_index_size_bytes", 2048, unit="bytes")
store.put_count_node_level("rally-node-1", "final_index_size_bytes", 4096, unit="bytes")
stats = reporter.calculate_results(store, metrics.create_race(cfg, t, challenge))
del store
opm = stats.metrics("index #1")
self.assertEqual(collections.OrderedDict([("min", 500), ("median", 1000), ("max", 2000), ("unit", "docs/s")]), opm["throughput"])
self.assertEqual(collections.OrderedDict([("50_0", 220), ("100_0", 225)]), opm["latency"])
self.assertEqual(collections.OrderedDict([("50_0", 200), ("100_0", 215)]), opm["service_time"])
self.assertAlmostEqual(0.3333333333333333, opm["error_rate"])
self.assertEqual(6144, stats.index_size)
def test_search_task_two_clients(self):
task = track.Task(track.Operation("search", track.OperationType.Search.name, param_source="driver-test-param-source"),
warmup_iterations=2, iterations=10, clients=2, target_throughput=10)
schedule = driver.schedule_for(self.test_track, task, 0)
expected_schedule = [
(0, metrics.SampleType.Warmup, 0, 6, None, {}),
(0.2, metrics.SampleType.Normal, 0, 6, None, {}),
(0.4, metrics.SampleType.Normal, 1, 6, None, {}),
(0.6, metrics.SampleType.Normal, 2, 6, None, {}),
(0.8, metrics.SampleType.Normal, 3, 6, None, {}),
(1.0, metrics.SampleType.Normal, 4, 6, None, {}),
]
self.assert_schedule(expected_schedule, schedule)
def test_finite_schedule_with_progress_indication(self):
task = track.Task("time-based", track.Operation("time-based", track.OperationType.Bulk.name, params={"body": ["a"], "size": 5},
param_source="driver-test-param-source"),
warmup_time_period=0, clients=4, params={"target-throughput": 4, "clients": 4})
invocations = driver.schedule_for(self.test_track, task, 0)
self.assert_schedule([
(0.0, metrics.SampleType.Normal, 1 / 5, {"body": ["a"], "size": 5}),
(1.0, metrics.SampleType.Normal, 2 / 5, {"body": ["a"], "size": 5}),
(2.0, metrics.SampleType.Normal, 3 / 5, {"body": ["a"], "size": 5}),
(3.0, metrics.SampleType.Normal, 4 / 5, {"body": ["a"], "size": 5}),
(4.0, metrics.SampleType.Normal, 5 / 5, {"body": ["a"], "size": 5}),
], list(invocations), infinite_schedule=False)
def test_schedule_param_source_determines_iterations_including_warmup(self):
task = track.Task("bulk-index", track.Operation("bulk-index", track.OperationType.Bulk.name, params={"body": ["a"], "size": 5},
param_source="driver-test-param-source"),
warmup_iterations=2, clients=1, params={"target-throughput": 4, "clients": 4})
invocations = driver.schedule_for(self.test_track, task, 0)
self.assert_schedule([
(0.0, metrics.SampleType.Warmup, 1 / 5, {"body": ["a"], "size": 5}),
(1.0, metrics.SampleType.Warmup, 2 / 5, {"body": ["a"], "size": 5}),
(2.0, metrics.SampleType.Normal, 3 / 5, {"body": ["a"], "size": 5}),
(3.0, metrics.SampleType.Normal, 4 / 5, {"body": ["a"], "size": 5}),
(4.0, metrics.SampleType.Normal, 5 / 5, {"body": ["a"], "size": 5}),
], list(invocations))
def test_search_task_two_clients(self):
task = track.Task(track.Operation("search", track.OperationType.Search.name, param_source="driver-test-param-source"),
warmup_iterations=2, iterations=10, clients=2, params={"target-throughput": 10, "clients": 2})
schedule = driver.schedule_for(self.test_track, task, 0)
expected_schedule = [
(0, metrics.SampleType.Warmup, 1 / 6, {}),
(0.2, metrics.SampleType.Normal, 2 / 6, {}),
(0.4, metrics.SampleType.Normal, 3 / 6, {}),
(0.6, metrics.SampleType.Normal, 4 / 6, {}),
(0.8, metrics.SampleType.Normal, 5 / 6, {}),
(1.0, metrics.SampleType.Normal, 6 / 6, {}),
]
self.assert_schedule(expected_schedule, schedule)
def test_allocates_two_parallel_tasks(self):
op = track.Operation("index", track.OperationType.Index, param_source="driver-test-param-source")
task = track.Task(op)
allocator = driver.Allocator([track.Parallel([task, task])])
self.assertEqual(2, allocator.clients)
self.assertEqual(3, len(allocator.allocations[0]))
self.assertEqual(3, len(allocator.allocations[1]))
self.assertEqual(2, len(allocator.join_points))
self.assertEqual([{op}], allocator.operations_per_joinpoint)
for join_point in allocator.join_points:
self.assertFalse(join_point.preceding_task_completes_parent)
self.assertEqual(0, join_point.num_clients_executing_completing_task)
def test_allocates_more_tasks_than_clients(self):
op1 = track.Operation("index-a", track.OperationType.Index)
op2 = track.Operation("index-b", track.OperationType.Index)
op3 = track.Operation("index-c", track.OperationType.Index)
op4 = track.Operation("index-d", track.OperationType.Index)
op5 = track.Operation("index-e", track.OperationType.Index)
index_a = track.Task(op1)
index_b = track.Task(op2)
index_c = track.Task(op3)
index_d = track.Task(op4)
index_e = track.Task(op5)
allocator = driver.Allocator([
track.Parallel(tasks=[index_a, index_b, index_c, index_d, index_e],
clients=2)
])
self.assertEqual(2, allocator.clients)
allocations = allocator.allocations
self.assertEqual(2, len(allocations))
# join_point, index_a, index_c, index_e, join_point
self.assertEqual(5, len(allocations[0]))
# we really have no chance to extract the join point so we just take what is there...
self.assertEqual(
[allocations[0][0], index_a, index_c, index_e, allocations[0][4]],
allocations[0])
# join_point, index_a, index_c, None, join_point
self.assertEqual(5, len(allocator.allocations[1]))
self.assertEqual(
[allocations[1][0], index_b, index_d, None, allocations[1][4]],
allocations[1])
self.assertEqual([{op1, op2, op3, op4, op5}],
allocator.operations_per_joinpoint)
def test_execute_schedule_in_throughput_mode(self, es):
es.bulk.return_value = {"errors": False}
params.register_param_source_for_name("driver-test-param-source",
DriverTestParamSource)
test_track = track.Track(name="unittest",
short_description="unittest track",
description="unittest track",
source_root_url="http://example.org",
indices=None,
challenges=None)
task = track.Task(track.Operation(
"time-based",
track.OperationType.Index.name,
params={
"body": ["action_metadata_line", "index_line"],
"action_metadata_present": True
},
param_source="driver-test-param-source"),
warmup_time_period=0,
clients=4,
target_throughput=None)
schedule = driver.schedule_for(test_track, task, 0)
sampler = driver.Sampler(client_id=2, task=task, start_timestamp=100)
cancel = threading.Event()
driver.execute_schedule(cancel, 0, task.operation, schedule, es,
sampler)
samples = sampler.samples
self.assertTrue(len(samples) > 0)
previous_absolute_time = -1.0
previous_relative_time = -1.0
for sample in samples:
self.assertEqual(2, sample.client_id)
self.assertEqual(task, sample.task)
self.assertTrue(previous_absolute_time < sample.absolute_time)
previous_absolute_time = sample.absolute_time
self.assertTrue(previous_relative_time < sample.relative_time)
previous_relative_time = sample.relative_time
# we don't have any warmup time period
self.assertEqual(metrics.SampleType.Normal, sample.sample_type)
# latency equals service time in throughput mode
self.assertEqual(sample.latency_ms, sample.service_time_ms)
self.assertEqual(1, sample.total_ops)
self.assertEqual("docs", sample.total_ops_unit)
self.assertEqual(1, sample.request_meta_data["bulk-size"])