Esempio n. 1
0
    def test_allocates_mixed_tasks(self):
        op1 = track.Operation("index",
                              track.OperationType.Index,
                              param_source="driver-test-param-source")
        op2 = track.Operation("stats",
                              track.OperationType.IndicesStats,
                              param_source="driver-test-param-source")
        op3 = track.Operation("search",
                              track.OperationType.Search,
                              param_source="driver-test-param-source")

        index = track.Task(op1)
        stats = track.Task(op2)
        search = track.Task(op3)

        allocator = driver.Allocator([
            index,
            track.Parallel([index, stats, stats]), index, index,
            track.Parallel([search, search, search])
        ])

        self.assertEqual(3, allocator.clients)

        # 1 join point, 1 op, 1 jp, 1 (parallel) op, 1 jp, 1 op, 1 jp, 1 op, 1 jp, 1 (parallel) op, 1 jp
        self.assertEqual(11, len(allocator.allocations[0]))
        self.assertEqual(11, len(allocator.allocations[1]))
        self.assertEqual(11, len(allocator.allocations[2]))
        self.assertEqual(6, len(allocator.join_points))
        self.assertEqual([{op1}, {op1, op2}, {op1}, {op1}, {op3}],
                         allocator.operations_per_joinpoint)
Esempio n. 2
0
    def test_allocates_more_tasks_than_clients(self):
        op1 = track.Operation("index-a", track.OperationType.Index, param_source="driver-test-param-source")
        op2 = track.Operation("index-b", track.OperationType.Index, param_source="driver-test-param-source")
        op3 = track.Operation("index-c", track.OperationType.Index, param_source="driver-test-param-source")
        op4 = track.Operation("index-d", track.OperationType.Index, param_source="driver-test-param-source")
        op5 = track.Operation("index-e", track.OperationType.Index, param_source="driver-test-param-source")

        index_a = track.Task(op1)
        index_b = track.Task(op2, completes_parent=True)
        index_c = track.Task(op3)
        index_d = track.Task(op4)
        index_e = track.Task(op5)

        allocator = driver.Allocator([track.Parallel(tasks=[index_a, index_b, index_c, index_d, index_e], clients=2)])

        self.assertEqual(2, allocator.clients)

        allocations = allocator.allocations

        # 2 clients
        self.assertEqual(2, len(allocations))
        # join_point, index_a, index_c, index_e, join_point
        self.assertEqual(5, len(allocations[0]))
        # we really have no chance to extract the join point so we just take what is there...
        self.assertEqual([allocations[0][0], index_a, index_c, index_e, allocations[0][4]], allocations[0])
        # join_point, index_a, index_c, None, join_point
        self.assertEqual(5, len(allocator.allocations[1]))
        self.assertEqual([allocations[1][0], index_b, index_d, None, allocations[1][4]], allocations[1])

        self.assertEqual([{op1, op2, op3, op4, op5}], allocator.operations_per_joinpoint)
        self.assertEqual(2, len(allocator.join_points))
        final_join_point = allocator.join_points[1]
        self.assertTrue(final_join_point.preceding_task_completes_parent)
        self.assertEqual(1, final_join_point.num_clients_executing_completing_task)
        self.assertEqual([1], final_join_point.clients_executing_completing_task)
Esempio n. 3
0
    def test_considers_number_of_clients_per_subtask(self):
        op1 = track.Operation("index-a", track.OperationType.Index, param_source="driver-test-param-source")
        op2 = track.Operation("index-b", track.OperationType.Index, param_source="driver-test-param-source")
        op3 = track.Operation("index-c", track.OperationType.Index, param_source="driver-test-param-source")

        index_a = track.Task(op1)
        index_b = track.Task(op2)
        index_c = track.Task(op3, clients=2)

        allocator = driver.Allocator([track.Parallel(tasks=[index_a, index_b, index_c], clients=3)])

        self.assertEqual(3, allocator.clients)

        allocations = allocator.allocations

        self.assertEqual(3, len(allocations))
        # join_point, index_a, index_c, join_point
        self.assertEqual(4, len(allocations[0]))
        # we really have no chance to extract the join point so we just take what is there...
        self.assertEqual([allocations[0][0], index_a, index_c, allocations[0][3]], allocations[0])
        # join_point, index_b, None, join_point
        self.assertEqual(4, len(allocator.allocations[1]))
        self.assertEqual([allocations[1][0], index_b, None, allocations[1][3]], allocations[1])

        self.assertEqual(4, len(allocator.allocations[2]))
        self.assertEqual([allocations[2][0], index_c, None, allocations[2][3]], allocations[2])

        self.assertEqual([{op1, op2, op3}], allocator.operations_per_joinpoint)
Esempio n. 4
0
    def test_considers_number_of_clients_per_subtask(self):
        op1 = track.Operation("index-a",
                              track.OperationType.Index,
                              param_source="driver-test-param-source")
        op2 = track.Operation("index-b",
                              track.OperationType.Index,
                              param_source="driver-test-param-source")
        op3 = track.Operation("index-c",
                              track.OperationType.Index,
                              param_source="driver-test-param-source")

        index_a = track.Task(op1)
        index_b = track.Task(op2)
        index_c = track.Task(op3, clients=2, completes_parent=True)

        allocator = driver.Allocator(
            [track.Parallel(tasks=[index_a, index_b, index_c], clients=3)])

        self.assertEqual(3, allocator.clients)

        allocations = allocator.allocations

        # 3 clients
        self.assertEqual(3, len(allocations))
        # join_point, index_a, index_c, join_point
        self.assertEqual(4, len(allocations[0]))
        # we really have no chance to extract the join point so we just take what is there...
        self.assertEqual(
            [allocations[0][0], index_a, index_c, allocations[0][3]],
            allocations[0])
        # join_point, index_b, None, join_point
        self.assertEqual(4, len(allocator.allocations[1]))
        self.assertEqual([allocations[1][0], index_b, None, allocations[1][3]],
                         allocations[1])

        self.assertEqual(4, len(allocator.allocations[2]))
        self.assertEqual([allocations[2][0], index_c, None, allocations[2][3]],
                         allocations[2])

        self.assertEqual([{op1, op2, op3}], allocator.operations_per_joinpoint)

        self.assertEqual(2, len(allocator.join_points))
        final_join_point = allocator.join_points[1]
        self.assertTrue(final_join_point.preceding_task_completes_parent)
        # task index_c has two clients, hence we have to wait for two clients to finish
        self.assertEqual(
            2, final_join_point.num_clients_executing_completing_task)
        self.assertEqual([2, 0],
                         final_join_point.clients_executing_completing_task)
Esempio n. 5
0
    def test_cancel_execute_schedule(self, es):
        es.bulk.return_value = {
            "errors": False
        }

        params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
        test_track = track.Track(name="unittest", short_description="unittest track",
                                 source_root_url="http://example.org",
                                 indices=None,
                                 challenges=None)

        # in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples
        for target_throughput, bounds in {10: [2, 4], 100: [24, 26], 1000: [245, 255]}.items():
            task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={
                "body": ["action_metadata_line", "index_line"],
                "action_metadata_present": True,
                "bulk-size": 1
            },
                                              param_source="driver-test-param-source"),
                              warmup_time_period=0.5, time_period=0.5, clients=4,
                              params={"target-throughput": target_throughput, "clients": 4})
            schedule = driver.schedule_for(test_track, task, 0)
            sampler = driver.Sampler(client_id=0, task=task, start_timestamp=0)

            cancel = threading.Event()
            complete = threading.Event()
            execute_schedule = driver.Executor(task, schedule, es, sampler, cancel, complete)

            cancel.set()
            execute_schedule()

            samples = sampler.samples

            sample_size = len(samples)
            self.assertEqual(0, sample_size)
Esempio n. 6
0
    def test_eternal_schedule(self):
        task = track.Task("time-based",
                          track.Operation(
                              "time-based",
                              track.OperationType.Bulk.name,
                              params={"body": ["a"]},
                              param_source="driver-test-param-source"),
                          warmup_time_period=0,
                          clients=4,
                          params={
                              "target-throughput": 4,
                              "clients": 4
                          })

        invocations = driver.schedule_for(self.test_track, task, 0)

        self.assert_schedule([
            (0.0, metrics.SampleType.Normal, None, {
                "body": ["a"]
            }),
            (1.0, metrics.SampleType.Normal, None, {
                "body": ["a"]
            }),
            (2.0, metrics.SampleType.Normal, None, {
                "body": ["a"]
            }),
            (3.0, metrics.SampleType.Normal, None, {
                "body": ["a"]
            }),
            (4.0, metrics.SampleType.Normal, None, {
                "body": ["a"]
            }),
        ],
                             invocations,
                             eternal_schedule=True)
Esempio n. 7
0
    def test_execute_schedule_throughput_throttled(self, es):
        es.bulk.return_value = {"errors": False}

        params.register_param_source_for_name("driver-test-param-source",
                                              DriverTestParamSource)
        test_track = track.Track(name="unittest",
                                 description="unittest track",
                                 indices=None,
                                 challenges=None)

        # in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples
        for target_throughput, bounds in {
                10: [2, 4],
                100: [24, 26],
                1000: [245, 255]
        }.items():
            task = track.Task("time-based",
                              track.Operation(
                                  "time-based",
                                  track.OperationType.Bulk.name,
                                  params={
                                      "body":
                                      ["action_metadata_line", "index_line"],
                                      "action_metadata_present":
                                      True,
                                      "bulk-size":
                                      1
                                  },
                                  param_source="driver-test-param-source"),
                              warmup_time_period=0.5,
                              time_period=0.5,
                              clients=4,
                              params={
                                  "target-throughput": target_throughput,
                                  "clients": 4
                              },
                              completes_parent=True)
            schedule = driver.schedule_for(test_track, task, 0)
            sampler = driver.Sampler(client_id=0, task=task, start_timestamp=0)

            cancel = threading.Event()
            complete = threading.Event()

            execute_schedule = driver.Executor(task, schedule, es, sampler,
                                               cancel, complete)
            execute_schedule()

            samples = sampler.samples

            sample_size = len(samples)
            lower_bound = bounds[0]
            upper_bound = bounds[1]
            self.assertTrue(
                lower_bound <= sample_size <= upper_bound,
                msg="Expected sample size to be between %d and %d but was %d" %
                (lower_bound, upper_bound, sample_size))
            self.assertTrue(
                complete.is_set(),
                "Executor should auto-complete a task that terminates its parent"
            )
Esempio n. 8
0
    def test_schedule_param_source_determines_iterations_no_warmup(self):
        # we neither define any time-period nor any iteration count on the task.
        task = track.Task("bulk-index",
                          track.Operation(
                              "bulk-index",
                              track.OperationType.Bulk.name,
                              params={
                                  "body": ["a"],
                                  "size": 3
                              },
                              param_source="driver-test-param-source"),
                          clients=1,
                          params={
                              "target-throughput": 4,
                              "clients": 4
                          })

        invocations = driver.schedule_for(self.test_track, task, 0)

        self.assert_schedule([
            (0.0, metrics.SampleType.Normal, 1 / 3, {
                "body": ["a"],
                "size": 3
            }),
            (1.0, metrics.SampleType.Normal, 2 / 3, {
                "body": ["a"],
                "size": 3
            }),
            (2.0, metrics.SampleType.Normal, 3 / 3, {
                "body": ["a"],
                "size": 3
            }),
        ], list(invocations))
Esempio n. 9
0
    def test_execute_schedule_aborts_on_error(self, es):
        class ExpectedUnitTestException(Exception):
            pass

        def run(*args, **kwargs):
            raise ExpectedUnitTestException()

        task = track.Task("no-op",
                          track.Operation(
                              "no-op",
                              track.OperationType.Bulk.name,
                              params={},
                              param_source="driver-test-param-source"),
                          warmup_time_period=0.5,
                          time_period=0.5,
                          clients=4,
                          params={"clients": 4})

        schedule = [(0, metrics.SampleType.Warmup, 0,
                     self.context_managed(run), None)]
        sampler = driver.Sampler(client_id=0, task=None, start_timestamp=0)
        cancel = threading.Event()
        complete = threading.Event()
        execute_schedule = driver.Executor(task, schedule, es, sampler, cancel,
                                           complete)

        with self.assertRaises(ExpectedUnitTestException):
            execute_schedule()

        es.assert_not_called()
Esempio n. 10
0
    def test_calculate_simple_index_stats(self):
        cfg = config.Config()
        cfg.add(config.Scope.application, "system", "env.name", "unittest")

        store = metrics.InMemoryMetricsStore(config=cfg, clear=True)
        store.open(datetime.datetime.now(), "test", "unittest", "unittest_car")

        store.put_value_cluster_level("throughput", 500, unit="docs/s", operation="index", operation_type=track.OperationType.Index)
        store.put_value_cluster_level("throughput", 1000, unit="docs/s", operation="index", operation_type=track.OperationType.Index)
        store.put_value_cluster_level("throughput", 2000, unit="docs/s", operation="index", operation_type=track.OperationType.Index)

        store.put_value_cluster_level("latency", 2800, unit="ms", operation="index", operation_type=track.OperationType.Index,
                                      sample_type=metrics.SampleType.Warmup)
        store.put_value_cluster_level("latency", 200, unit="ms", operation="index", operation_type=track.OperationType.Index)
        store.put_value_cluster_level("latency", 220, unit="ms", operation="index", operation_type=track.OperationType.Index)
        store.put_value_cluster_level("latency", 225, unit="ms", operation="index", operation_type=track.OperationType.Index)

        store.put_value_cluster_level("service_time", 250, unit="ms", operation="index", operation_type=track.OperationType.Index,
                                      sample_type=metrics.SampleType.Warmup)
        store.put_value_cluster_level("service_time", 190, unit="ms", operation="index", operation_type=track.OperationType.Index)
        store.put_value_cluster_level("service_time", 200, unit="ms", operation="index", operation_type=track.OperationType.Index)
        store.put_value_cluster_level("service_time", 215, unit="ms", operation="index", operation_type=track.OperationType.Index)

        index = track.Task(operation=track.Operation(name="index", operation_type=track.OperationType.Index, granularity_unit="docs/s"))
        challenge = track.Challenge(name="unittest", description="", index_settings=None, schedule=[index])

        stats = reporter.Stats(store, challenge)

        self.assertEqual((500, 1000, 2000, "docs/s"), stats.op_metrics["index"]["throughput"])
        self.assertEqual(collections.OrderedDict([(50.0, 220), (100, 225)]), stats.op_metrics["index"]["latency"])
        self.assertEqual(collections.OrderedDict([(50.0, 200), (100, 215)]), stats.op_metrics["index"]["service_time"])
Esempio n. 11
0
    def test_single_metrics_aggregation(self):
        op = track.Operation("index", track.OperationType.Index, param_source="driver-test-param-source")

        samples = [
            driver.Sample(0, 1470838595, 21, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 1, 1 / 9),
            driver.Sample(0, 1470838596, 22, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 2, 2 / 9),
            driver.Sample(0, 1470838597, 23, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 3, 3 / 9),
            driver.Sample(0, 1470838598, 24, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 4, 4 / 9),
            driver.Sample(0, 1470838599, 25, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 5, 5 / 9),
            driver.Sample(0, 1470838600, 26, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 6, 6 / 9),
            driver.Sample(1, 1470838598.5, 24.5, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 4.5, 7 / 9),
            driver.Sample(1, 1470838599.5, 25.5, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 5.5, 8 / 9),
            driver.Sample(1, 1470838600.5, 26.5, op, metrics.SampleType.Normal, None, -1, -1, 5000, "docs", 6.5, 9 / 9)
        ]

        aggregated = driver.calculate_global_throughput(samples)

        self.assertIn(op, aggregated)
        self.assertEqual(1, len(aggregated))

        throughput = aggregated[op]
        self.assertEqual(6, len(throughput))
        self.assertEqual((1470838595, 21, metrics.SampleType.Normal, 5000, "docs/s"), throughput[0])
        self.assertEqual((1470838596, 22, metrics.SampleType.Normal, 5000, "docs/s"), throughput[1])
        self.assertEqual((1470838597, 23, metrics.SampleType.Normal, 5000, "docs/s"), throughput[2])
        self.assertEqual((1470838598, 24, metrics.SampleType.Normal, 5000, "docs/s"), throughput[3])
        self.assertEqual((1470838599, 25, metrics.SampleType.Normal, 6000, "docs/s"), throughput[4])
        self.assertEqual((1470838600, 26, metrics.SampleType.Normal, 6666.666666666667, "docs/s"), throughput[5])
Esempio n. 12
0
    def test_search_task_one_client(self):
        task = track.Task("search",
                          track.Operation(
                              "search",
                              track.OperationType.Search.name,
                              param_source="driver-test-param-source"),
                          warmup_iterations=3,
                          iterations=5,
                          clients=1,
                          params={
                              "target-throughput": 10,
                              "clients": 1
                          })
        schedule = driver.schedule_for(self.test_track, task, 0)

        expected_schedule = [
            (0, metrics.SampleType.Warmup, 1 / 8, {}),
            (0.1, metrics.SampleType.Warmup, 2 / 8, {}),
            (0.2, metrics.SampleType.Warmup, 3 / 8, {}),
            (0.3, metrics.SampleType.Normal, 4 / 8, {}),
            (0.4, metrics.SampleType.Normal, 5 / 8, {}),
            (0.5, metrics.SampleType.Normal, 6 / 8, {}),
            (0.6, metrics.SampleType.Normal, 7 / 8, {}),
            (0.7, metrics.SampleType.Normal, 8 / 8, {}),
        ]
        self.assert_schedule(expected_schedule, schedule)
Esempio n. 13
0
    def test_schedule_for_time_based(self):
        task = track.Task("time-based", track.Operation("time-based", track.OperationType.Bulk.name, params={"body": ["a"], "size": 11},
                                                        param_source="driver-test-param-source"), warmup_time_period=0.1, time_period=0.1,
                          clients=1)

        schedule_handle = driver.schedule_for(self.test_track, task, 0)
        schedule = list(schedule_handle())

        self.assertTrue(len(schedule) > 0)

        last_progress = -1

        for invocation_time, sample_type, progress_percent, runner, params in schedule:
            # we're not throughput throttled
            self.assertEqual(0, invocation_time)
            if progress_percent <= 0.5:
                self.assertEqual(metrics.SampleType.Warmup, sample_type)
            else:
                self.assertEqual(metrics.SampleType.Normal, sample_type)
            self.assertTrue(last_progress < progress_percent)
            last_progress = progress_percent
            self.assertTrue(round(progress_percent, 2) >= 0.0, "progress should be >= 0.0 but was [%f]" % progress_percent)
            self.assertTrue(round(progress_percent, 2) <= 1.0, "progress should be <= 1.0 but was [%f]" % progress_percent)
            self.assertIsNotNone(runner, "runner must be defined")
            self.assertEqual({"body": ["a"], "size": 11}, params)
Esempio n. 14
0
    def test_execute_schedule_throughput_throttled(self, es):
        es.bulk.return_value = {
            "errors": False
        }

        params.register_param_source_for_name("driver-test-param-source", DriverTestParamSource)
        test_track = track.Track(name="unittest", short_description="unittest track", description="unittest track",
                                 source_root_url="http://example.org",
                                 indices=None,
                                 challenges=None)

        # in one second (0.5 warmup + 0.5 measurement) we should get 1000 [ops/s] / 4 [clients] = 250 samples
        for target_throughput, bounds in {10: [2, 4], 100: [24, 26], 1000: [245, 255]}.items():
            task = track.Task(track.Operation("time-based", track.OperationType.Index.name, params={
                "body": ["action_metadata_line", "index_line"],
                "action_metadata_present": True
            },
                                              param_source="driver-test-param-source"),
                              warmup_time_period=0.5, time_period=0.5, clients=4, target_throughput=target_throughput)
            schedule = driver.schedule_for(test_track, task, 0)
            sampler = driver.Sampler(client_id=0, operation=task.operation, start_timestamp=0)

            driver.execute_schedule(schedule, es, sampler)

            samples = sampler.samples

            sample_size = len(samples)
            lower_bound = bounds[0]
            upper_bound = bounds[1]
            self.assertTrue(lower_bound <= sample_size <= upper_bound,
                            msg="Expected sample size to be between %d and %d but was %d" % (lower_bound, upper_bound, sample_size))
Esempio n. 15
0
    def test_store_race(self):
        self.cfg.add(config.Scope.application, "system", "pipeline", "unittest-pipeline")
        self.cfg.add(config.Scope.application, "system", "user.tag", "")
        self.cfg.add(config.Scope.application, "benchmarks", "challenge", "index-and-search")
        self.cfg.add(config.Scope.application, "benchmarks", "car", "defaults")
        self.cfg.add(config.Scope.application, "benchmarks", "laps", 1)
        self.cfg.add(config.Scope.application, "launcher", "external.target.hosts", [{"host": "localhost", "port": "9200"}])
        self.cfg.add(config.Scope.application, "source", "revision", "latest")
        self.cfg.add(config.Scope.application, "source", "distribution.version", "5.0.0")

        index = "tests"
        type = "test-type"

        schedule = [
            track.Task(track.Operation("index", track.OperationType.Index, None)),
            track.Task(track.Operation("search-all", track.OperationType.Search, None)),
        ]

        t = track.Track(name="unittest", short_description="unittest track", description="unittest track",
                        source_root_url="http://example.org",
                        indices=[track.Index(name=index, types=[track.Type(name=type, mapping_file=None)])],
                        challenges=[
                            track.Challenge(name="index-and-search", description="Index & Search", index_settings=None, schedule=schedule)
                        ])
        self.race_store.store_race(t)

        expected_doc = {
            "environment": "unittest-env",
            "trial-timestamp": "20160131T000000Z",
            "pipeline": "unittest-pipeline",
            "revision": "latest",
            "distribution-version": "5.0.0",
            "track": "unittest",
            "laps": 1,
            "selected-challenge": {
                "name": "index-and-search",
                "operations": [
                    "index",
                    "search-all"
                ]
            },
            "car": "defaults",
            "target-hosts": ["localhost:9200"],
            "user-tag": ""
        }

        self.es_mock.index.assert_called_with(index="rally-2016", doc_type="races", item=expected_doc)
Esempio n. 16
0
    def test_a_task_completes_the_parallel_structure(self):
        opA = track.Operation("index-completing", track.OperationType.Index, param_source="driver-test-param-source")
        opB = track.Operation("index-non-completing", track.OperationType.Index, param_source="driver-test-param-source")
        taskA = track.Task(opA, completes_parent=True)
        taskB = track.Task(opB)

        allocator = driver.Allocator([track.Parallel([taskA, taskB])])

        self.assertEqual(2, allocator.clients)
        self.assertEqual(3, len(allocator.allocations[0]))
        self.assertEqual(3, len(allocator.allocations[1]))
        self.assertEqual(2, len(allocator.join_points))
        self.assertEqual([{opA, opB}], allocator.operations_per_joinpoint)
        final_join_point = allocator.join_points[1]
        self.assertTrue(final_join_point.preceding_task_completes_parent)
        self.assertEqual(1, final_join_point.num_clients_executing_completing_task)
        self.assertEqual([0], final_join_point.clients_executing_completing_task)
Esempio n. 17
0
    def test_execute_schedule_in_throughput_mode(self, es):
        es.bulk.return_value = {"errors": False}

        params.register_param_source_for_name("driver-test-param-source",
                                              DriverTestParamSource)
        test_track = track.Track(name="unittest",
                                 description="unittest track",
                                 indices=None,
                                 challenges=None)

        task = track.Task(
            "time-based",
            track.Operation(
                "time-based",
                track.OperationType.Bulk.name,
                params={
                    "body": ["action_metadata_line", "index_line"],
                    "action-metadata-present": True,
                    "bulk-size": 1,
                    # we need this because DriverTestParamSource does not know that we only have one bulk and hence size() returns incorrect results
                    "size": 1
                },
                param_source="driver-test-param-source"),
            warmup_time_period=0,
            clients=4)
        schedule = driver.schedule_for(test_track, task, 0)

        sampler = driver.Sampler(client_id=2,
                                 task=task,
                                 start_timestamp=time.perf_counter())
        cancel = threading.Event()
        complete = threading.Event()

        execute_schedule = driver.Executor(task, schedule, es, sampler, cancel,
                                           complete)
        execute_schedule()

        samples = sampler.samples

        self.assertTrue(len(samples) > 0)
        self.assertFalse(complete.is_set(),
                         "Executor should not auto-complete a normal task")
        previous_absolute_time = -1.0
        previous_relative_time = -1.0
        for sample in samples:
            self.assertEqual(2, sample.client_id)
            self.assertEqual(task, sample.task)
            self.assertLess(previous_absolute_time, sample.absolute_time)
            previous_absolute_time = sample.absolute_time
            self.assertLess(previous_relative_time, sample.relative_time)
            previous_relative_time = sample.relative_time
            # we don't have any warmup time period
            self.assertEqual(metrics.SampleType.Normal, sample.sample_type)
            # latency equals service time in throughput mode
            self.assertEqual(sample.latency_ms, sample.service_time_ms)
            self.assertEqual(1, sample.total_ops)
            self.assertEqual("docs", sample.total_ops_unit)
            self.assertEqual(1, sample.request_meta_data["bulk-size"])
Esempio n. 18
0
    def test_store_race(self):
        from esrally import time
        schedule = [
            track.Task("index #1",
                       track.Operation("index", track.OperationType.Bulk))
        ]

        t = track.Track(
            name="unittest",
            indices=[track.Index(name="tests", types=["test-type"])],
            challenges=[
                track.Challenge(name="index", default=True, schedule=schedule)
            ])

        race = metrics.Race(rally_version="0.4.4",
                            environment_name="unittest",
                            trial_id=FileRaceStoreTests.TRIAL_ID,
                            trial_timestamp=FileRaceStoreTests.TRIAL_TIMESTAMP,
                            pipeline="from-sources",
                            user_tags={"os": "Linux"},
                            track=t,
                            track_params={"clients": 12},
                            challenge=t.default_challenge,
                            car="4gheap",
                            total_laps=12,
                            cluster=FileRaceStoreTests.DictHolder({
                                "distribution-version":
                                "5.0.0",
                                "nodes": [{
                                    "node_name": "node0",
                                    "ip": "127.0.0.1"
                                }]
                            }),
                            lap_results=[],
                            results=FileRaceStoreTests.DictHolder({
                                "young_gc_time":
                                100,
                                "old_gc_time":
                                5,
                                "op_metrics": [{
                                    "task": "index #1",
                                    "operation": "index",
                                    "throughput": {
                                        "min": 1000,
                                        "median": 1250,
                                        "max": 1500,
                                        "unit": "docs/s"
                                    }
                                }]
                            }))

        self.race_store.store_race(race)

        retrieved_race = self.race_store.find_by_timestamp(
            timestamp=time.to_iso8601(FileRaceStoreTests.TRIAL_TIMESTAMP))
        self.assertEqual(race.trial_timestamp, retrieved_race.trial_timestamp)
        self.assertEqual(1, len(self.race_store.list()))
Esempio n. 19
0
    def test_allocates_one_task(self):
        op = track.Operation("index", track.OperationType.Index, param_source="driver-test-param-source")
        task = track.Task(op)

        allocator = driver.Allocator([task])

        self.assertEqual(1, allocator.clients)
        self.assertEqual(3, len(allocator.allocations[0]))
        self.assertEqual(2, len(allocator.join_points))
        self.assertEqual([{op}], allocator.operations_per_joinpoint)
Esempio n. 20
0
    def test_allocates_two_parallel_tasks(self):
        op = track.Operation("index", track.OperationType.Index)
        task = track.Task(op)

        allocator = driver.Allocator([track.Parallel([task, task])])

        self.assertEqual(2, allocator.clients)
        self.assertEqual(3, len(allocator.allocations[0]))
        self.assertEqual(3, len(allocator.allocations[1]))
        self.assertEqual(2, len(allocator.join_points))
        self.assertEqual([{op}], allocator.operations_per_joinpoint)
Esempio n. 21
0
    def test_schedule_defaults_to_iteration_based(self):
        # no time-period and no iterations specified on the task. Also, the parameter source does not define a size.
        task = track.Task("bulk-index", track.Operation("bulk-index", track.OperationType.Bulk.name, params={"body": ["a"]},
                                                        param_source="driver-test-param-source"),
                          clients=1, params={"target-throughput": 4, "clients": 4})

        invocations = driver.schedule_for(self.test_track, task, 0)

        self.assert_schedule([
            (0.0, metrics.SampleType.Normal, 1 / 1, {"body": ["a"]}),
        ], list(invocations))
Esempio n. 22
0
    def test_allocates_two_serial_tasks(self):
        op = track.Operation("index", track.OperationType.Index, param_source="driver-test-param-source")
        task = track.Task(op)

        allocator = driver.Allocator([task, task])

        self.assertEqual(1, allocator.clients)
        # we have two operations and three join points
        self.assertEqual(5, len(allocator.allocations[0]))
        self.assertEqual(3, len(allocator.join_points))
        self.assertEqual([{op}, {op}], allocator.operations_per_joinpoint)
Esempio n. 23
0
    def test_calculate_simple_index_stats(self):
        cfg = config.Config()
        cfg.add(config.Scope.application, "system", "env.name", "unittest")
        cfg.add(config.Scope.application, "system", "time.start", datetime.datetime.now())
        cfg.add(config.Scope.application, "system", "trial.id", "6ebc6e53-ee20-4b0c-99b4-09697987e9f4")
        cfg.add(config.Scope.application, "reporting", "datastore.type", "in-memory")
        cfg.add(config.Scope.application, "mechanic", "car.names", ["unittest_car"])
        cfg.add(config.Scope.application, "mechanic", "car.params", {})
        cfg.add(config.Scope.application, "mechanic", "plugin.params", {})
        cfg.add(config.Scope.application, "race", "laps", 1)
        cfg.add(config.Scope.application, "race", "user.tag", "")
        cfg.add(config.Scope.application, "race", "pipeline", "from-sources-skip-build")
        cfg.add(config.Scope.application, "track", "params", {})

        index = track.Task(name="index #1", operation=track.Operation(name="index", operation_type=track.OperationType.Bulk, params=None))
        challenge = track.Challenge(name="unittest", schedule=[index], default=True)
        t = track.Track("unittest", "unittest-track", challenges=[challenge])

        store = metrics.metrics_store(cfg, read_only=False, track=t, challenge=challenge)
        store.lap = 1

        store.put_value_cluster_level("throughput", 500, unit="docs/s", task="index #1", operation_type=track.OperationType.Bulk)
        store.put_value_cluster_level("throughput", 1000, unit="docs/s", task="index #1", operation_type=track.OperationType.Bulk)
        store.put_value_cluster_level("throughput", 2000, unit="docs/s", task="index #1", operation_type=track.OperationType.Bulk)

        store.put_value_cluster_level("latency", 2800, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
                                      sample_type=metrics.SampleType.Warmup)
        store.put_value_cluster_level("latency", 200, unit="ms", task="index #1", operation_type=track.OperationType.Bulk)
        store.put_value_cluster_level("latency", 220, unit="ms", task="index #1", operation_type=track.OperationType.Bulk)
        store.put_value_cluster_level("latency", 225, unit="ms", task="index #1", operation_type=track.OperationType.Bulk)

        store.put_value_cluster_level("service_time", 250, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
                                      sample_type=metrics.SampleType.Warmup, meta_data={"success": False})
        store.put_value_cluster_level("service_time", 190, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
                                      meta_data={"success": True})
        store.put_value_cluster_level("service_time", 200, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
                                      meta_data={"success": False})
        store.put_value_cluster_level("service_time", 215, unit="ms", task="index #1", operation_type=track.OperationType.Bulk,
                                      meta_data={"success": True})
        store.put_count_node_level("rally-node-0", "final_index_size_bytes", 2048, unit="bytes")
        store.put_count_node_level("rally-node-1", "final_index_size_bytes", 4096, unit="bytes")

        stats = reporter.calculate_results(store, metrics.create_race(cfg, t, challenge))

        del store

        opm = stats.metrics("index #1")
        self.assertEqual(collections.OrderedDict([("min", 500), ("median", 1000), ("max", 2000), ("unit", "docs/s")]), opm["throughput"])
        self.assertEqual(collections.OrderedDict([("50_0", 220), ("100_0", 225)]), opm["latency"])
        self.assertEqual(collections.OrderedDict([("50_0", 200), ("100_0", 215)]), opm["service_time"])
        self.assertAlmostEqual(0.3333333333333333, opm["error_rate"])

        self.assertEqual(6144, stats.index_size)
Esempio n. 24
0
    def test_search_task_two_clients(self):
        task = track.Task(track.Operation("search", track.OperationType.Search.name, param_source="driver-test-param-source"),
                          warmup_iterations=2, iterations=10, clients=2, target_throughput=10)
        schedule = driver.schedule_for(self.test_track, task, 0)

        expected_schedule = [
            (0, metrics.SampleType.Warmup, 0, 6, None, {}),
            (0.2, metrics.SampleType.Normal, 0, 6, None, {}),
            (0.4, metrics.SampleType.Normal, 1, 6, None, {}),
            (0.6, metrics.SampleType.Normal, 2, 6, None, {}),
            (0.8, metrics.SampleType.Normal, 3, 6, None, {}),
            (1.0, metrics.SampleType.Normal, 4, 6, None, {}),
        ]
        self.assert_schedule(expected_schedule, schedule)
Esempio n. 25
0
    def test_finite_schedule_with_progress_indication(self):
        task = track.Task("time-based", track.Operation("time-based", track.OperationType.Bulk.name, params={"body": ["a"], "size": 5},
                                                        param_source="driver-test-param-source"),
                          warmup_time_period=0, clients=4, params={"target-throughput": 4, "clients": 4})

        invocations = driver.schedule_for(self.test_track, task, 0)

        self.assert_schedule([
            (0.0, metrics.SampleType.Normal, 1 / 5, {"body": ["a"], "size": 5}),
            (1.0, metrics.SampleType.Normal, 2 / 5, {"body": ["a"], "size": 5}),
            (2.0, metrics.SampleType.Normal, 3 / 5, {"body": ["a"], "size": 5}),
            (3.0, metrics.SampleType.Normal, 4 / 5, {"body": ["a"], "size": 5}),
            (4.0, metrics.SampleType.Normal, 5 / 5, {"body": ["a"], "size": 5}),
        ], list(invocations), infinite_schedule=False)
Esempio n. 26
0
    def test_schedule_param_source_determines_iterations_including_warmup(self):
        task = track.Task("bulk-index", track.Operation("bulk-index", track.OperationType.Bulk.name, params={"body": ["a"], "size": 5},
                                                        param_source="driver-test-param-source"),
                          warmup_iterations=2, clients=1, params={"target-throughput": 4, "clients": 4})

        invocations = driver.schedule_for(self.test_track, task, 0)

        self.assert_schedule([
            (0.0, metrics.SampleType.Warmup, 1 / 5, {"body": ["a"], "size": 5}),
            (1.0, metrics.SampleType.Warmup, 2 / 5, {"body": ["a"], "size": 5}),
            (2.0, metrics.SampleType.Normal, 3 / 5, {"body": ["a"], "size": 5}),
            (3.0, metrics.SampleType.Normal, 4 / 5, {"body": ["a"], "size": 5}),
            (4.0, metrics.SampleType.Normal, 5 / 5, {"body": ["a"], "size": 5}),
        ], list(invocations))
Esempio n. 27
0
    def test_search_task_two_clients(self):
        task = track.Task(track.Operation("search", track.OperationType.Search.name, param_source="driver-test-param-source"),
                          warmup_iterations=2, iterations=10, clients=2, params={"target-throughput": 10, "clients": 2})
        schedule = driver.schedule_for(self.test_track, task, 0)

        expected_schedule = [
            (0, metrics.SampleType.Warmup, 1 / 6, {}),
            (0.2, metrics.SampleType.Normal, 2 / 6, {}),
            (0.4, metrics.SampleType.Normal, 3 / 6, {}),
            (0.6, metrics.SampleType.Normal, 4 / 6, {}),
            (0.8, metrics.SampleType.Normal, 5 / 6, {}),
            (1.0, metrics.SampleType.Normal, 6 / 6, {}),
        ]
        self.assert_schedule(expected_schedule, schedule)
Esempio n. 28
0
    def test_allocates_two_parallel_tasks(self):
        op = track.Operation("index", track.OperationType.Index, param_source="driver-test-param-source")
        task = track.Task(op)

        allocator = driver.Allocator([track.Parallel([task, task])])

        self.assertEqual(2, allocator.clients)
        self.assertEqual(3, len(allocator.allocations[0]))
        self.assertEqual(3, len(allocator.allocations[1]))
        self.assertEqual(2, len(allocator.join_points))
        self.assertEqual([{op}], allocator.operations_per_joinpoint)
        for join_point in allocator.join_points:
            self.assertFalse(join_point.preceding_task_completes_parent)
            self.assertEqual(0, join_point.num_clients_executing_completing_task)
Esempio n. 29
0
    def test_allocates_more_tasks_than_clients(self):
        op1 = track.Operation("index-a", track.OperationType.Index)
        op2 = track.Operation("index-b", track.OperationType.Index)
        op3 = track.Operation("index-c", track.OperationType.Index)
        op4 = track.Operation("index-d", track.OperationType.Index)
        op5 = track.Operation("index-e", track.OperationType.Index)

        index_a = track.Task(op1)
        index_b = track.Task(op2)
        index_c = track.Task(op3)
        index_d = track.Task(op4)
        index_e = track.Task(op5)

        allocator = driver.Allocator([
            track.Parallel(tasks=[index_a, index_b, index_c, index_d, index_e],
                           clients=2)
        ])

        self.assertEqual(2, allocator.clients)

        allocations = allocator.allocations

        self.assertEqual(2, len(allocations))
        # join_point, index_a, index_c, index_e, join_point
        self.assertEqual(5, len(allocations[0]))
        # we really have no chance to extract the join point so we just take what is there...
        self.assertEqual(
            [allocations[0][0], index_a, index_c, index_e, allocations[0][4]],
            allocations[0])
        # join_point, index_a, index_c, None, join_point
        self.assertEqual(5, len(allocator.allocations[1]))
        self.assertEqual(
            [allocations[1][0], index_b, index_d, None, allocations[1][4]],
            allocations[1])

        self.assertEqual([{op1, op2, op3, op4, op5}],
                         allocator.operations_per_joinpoint)
Esempio n. 30
0
    def test_execute_schedule_in_throughput_mode(self, es):
        es.bulk.return_value = {"errors": False}

        params.register_param_source_for_name("driver-test-param-source",
                                              DriverTestParamSource)
        test_track = track.Track(name="unittest",
                                 short_description="unittest track",
                                 description="unittest track",
                                 source_root_url="http://example.org",
                                 indices=None,
                                 challenges=None)

        task = track.Task(track.Operation(
            "time-based",
            track.OperationType.Index.name,
            params={
                "body": ["action_metadata_line", "index_line"],
                "action_metadata_present": True
            },
            param_source="driver-test-param-source"),
                          warmup_time_period=0,
                          clients=4,
                          target_throughput=None)
        schedule = driver.schedule_for(test_track, task, 0)

        sampler = driver.Sampler(client_id=2, task=task, start_timestamp=100)
        cancel = threading.Event()
        driver.execute_schedule(cancel, 0, task.operation, schedule, es,
                                sampler)

        samples = sampler.samples

        self.assertTrue(len(samples) > 0)
        previous_absolute_time = -1.0
        previous_relative_time = -1.0
        for sample in samples:
            self.assertEqual(2, sample.client_id)
            self.assertEqual(task, sample.task)
            self.assertTrue(previous_absolute_time < sample.absolute_time)
            previous_absolute_time = sample.absolute_time
            self.assertTrue(previous_relative_time < sample.relative_time)
            previous_relative_time = sample.relative_time
            # we don't have any warmup time period
            self.assertEqual(metrics.SampleType.Normal, sample.sample_type)
            # latency equals service time in throughput mode
            self.assertEqual(sample.latency_ms, sample.service_time_ms)
            self.assertEqual(1, sample.total_ops)
            self.assertEqual("docs", sample.total_ops_unit)
            self.assertEqual(1, sample.request_meta_data["bulk-size"])