def __init__(self, benchmark_info, zipped_size=1000): """Setup initial values. :param benchmark_info: dict, generalized info about iterations. The most important value is `iterations_count' that should have int value of total data size :param zipped_size: int maximum number of points on scale """ self._data = costilius.OrderedDict() # Container for results self._benchmark_info = benchmark_info self.base_size = benchmark_info.get("iterations_count", 0) self.zipped_size = zipped_size
def __init__(self): self._atomic_actions = costilius.OrderedDict()
def test_optional_action_timer_decorator(self, mock_time): class TestAtomicTimer(atomic.ActionTimerMixin): @atomic.optional_action_timer("some") def some_func(self, a, b): return a + b @atomic.optional_action_timer("some", argument_name="foo", default=False) def other_func(self, a, b): return a + b inst = TestAtomicTimer() self.assertEqual(5, inst.some_func(2, 3)) self.assertEqual(costilius.OrderedDict({"some": 2}), inst.atomic_actions()) inst = TestAtomicTimer() self.assertEqual(5, inst.some_func(2, 3, atomic_action=False)) self.assertEqual(costilius.OrderedDict(), inst.atomic_actions()) inst = TestAtomicTimer() self.assertEqual(5, inst.other_func(2, 3)) self.assertEqual(costilius.OrderedDict(), inst.atomic_actions()) inst = TestAtomicTimer() self.assertEqual(5, inst.other_func(2, 3, foo=True)) self.assertEqual(costilius.OrderedDict({"some": 2}), inst.atomic_actions())
def test_add_iteration_and_render(self): self.assertRaises(TypeError, charts.Table, self.bench_info) table = self.Table(self.bench_info) self.assertEqual(costilius.OrderedDict(), table.render()) [table.add_iteration({"a": i, "b": 43 - i}) for i in range(1, 43)] self.assertEqual( costilius.OrderedDict([ ("value_a", costilius.OrderedDict([("foo", table.foo), ("bar", table.bar)])), ("value_b", costilius.OrderedDict([("foo", table.foo), ("bar", table.bar)])) ]), table.render())
def test_add_iteration_and_render(self): chart = charts.AtomicAvgChart({"iterations_count": 3, "atomic": {"foo": {}, "bar": {}}}) self.assertIsInstance(chart, charts.AvgChart) [chart.add_iteration({"atomic_actions": costilius.OrderedDict(a)}) for a in ([("foo", 2), ("bar", 5)], [("foo", 4)], [("bar", 7)])] self.assertEqual([("bar", 4.0), ("foo", 2.0)], sorted(chart.render()))
def test_add_iteration_and_render(self): chart = charts.AtomicHistogramChart( {"iterations_count": 3, "atomic": costilius.OrderedDict( [("foo", {"min_duration": 1.6, "max_duration": 2.8}), ("bar", {"min_duration": 3.1, "max_duration": 5.5})])}) self.assertIsInstance(chart, charts.HistogramChart) [chart.add_iteration({"atomic_actions": a}) for a in ({"foo": 1.6, "bar": 3.1}, {"foo": 2.8}, {"bar": 5.5})] expected = [ [{"disabled": 0, "key": "foo", "view": "Square Root Choice", "values": [{"x": 2.2, "y": 2}, {"x": 2.8, "y": 1}]}, {"disabled": 0, "key": "foo", "view": "Sturges Formula", "values": [{"x": 2.0, "y": 2}, {"x": 2.4, "y": 0}, {"x": 2.8, "y": 1}]}, {"disabled": 0, "key": "foo", "view": "Rice Rule", "values": [{"x": 2.0, "y": 2}, {"x": 2.4, "y": 0}, {"x": 2.8, "y": 1}]}], [{"disabled": 1, "key": "bar", "view": "Square Root Choice", "values": [{"x": 4.3, "y": 2}, {"x": 5.5, "y": 1}]}, {"disabled": 1, "key": "bar", "view": "Sturges Formula", "values": [{"x": 3.9, "y": 2}, {"x": 4.7, "y": 0}, {"x": 5.5, "y": 1}]}, {"disabled": 1, "key": "bar", "view": "Rice Rule", "values": [{"x": 3.9, "y": 2}, {"x": 4.7, "y": 0}, {"x": 5.5, "y": 1}]}]] self.assertEqual(expected, chart.render())
def test_add_iteration_and_render(self): table = charts.MainStatsTable({ "iterations_count": 42, "atomic": { "foo": {}, "bar": {} } }) [ table.add_iteration({ "atomic_actions": costilius.OrderedDict([("foo", i), ("bar", 43 - 1)]), "duration": i, "error": i % 40 }) for i in range(1, 43) ] expected_rows = [[ "foo", 1.0, 21.5, 38.5, 40.5, 42.0, 21.5, "100.0%", 42.0 ], ["bar", 42.0, 42.0, 42.0, 42.0, 42.0, 42.0, "100.0%", 42.0], ["total", 0.0, 0.0, 0.0, 0.0, 40.0, 0.952, "100.0%", 42.0]] self.assertEqual({ "cols": self.columns, "rows": expected_rows }, table.render())
def test_action_timer_decorator(self, mock_time): class Some(atomic.ActionTimerMixin): @atomic.action_timer("some") def some_func(self, a, b): return a + b inst = Some() self.assertEqual(5, inst.some_func(2, 3)) self.assertEqual(costilius.OrderedDict({"some": 2}), inst.atomic_actions())
def _init_columns(self): return costilius.OrderedDict( (("Min (sec)", streaming.MinComputation()), ("Median (sec)", streaming.PercentileComputation(50)), ("90%ile (sec)", streaming.PercentileComputation(90)), ("95%ile (sec)", streaming.PercentileComputation(95)), ("Max (sec)", streaming.MaxComputation()), ("Avg (sec)", streaming.MeanComputation()), ("Success", streaming.ProgressComputation(self.base_size)), ("Count", streaming.IncrementComputation())))
def test_action_timer_context(self, mock_time): inst = atomic.ActionTimerMixin() with atomic.ActionTimer(inst, "test"): with atomic.ActionTimer(inst, "test"): with atomic.ActionTimer(inst, "some"): pass expected = [("test", 20), ("test (2)", 12), ("some", 4)] self.assertEqual(costilius.OrderedDict(expected), inst.atomic_actions())
def test_action_timer_decorator_with_exception(self, mock_time): class TestException(Exception): pass class TestTimer(atomic.ActionTimerMixin): @atomic.action_timer("test") def some_func(self): raise TestException("test") inst = TestTimer() self.assertRaises(TestException, inst.some_func) self.assertEqual(costilius.OrderedDict({"test": 2}), inst.atomic_actions())
def test_action_timer_context_with_exception(self, mock_time): inst = atomic.ActionTimerMixin() class TestException(Exception): pass try: with atomic.ActionTimer(inst, "test"): raise TestException("test") except TestException: pass expected = [("test", 2)] self.assertEqual(costilius.OrderedDict(expected), inst.atomic_actions())
def get_atomic_actions_data(raw_data): """Retrieve detailed (by atomic actions & total runtime) benchmark data. :parameter raw_data: list of raw records (scenario runner output) :returns: dictionary containing atomic action + total duration lists for all atomic action keys """ atomic_actions = [] for row in raw_data: # find first non-error result to get atomic actions names if not row["error"] and "atomic_actions" in row: atomic_actions = row["atomic_actions"].keys() break actions_data = costilius.OrderedDict() for atomic_action in atomic_actions: actions_data[atomic_action] = [ r["atomic_actions"][atomic_action] for r in raw_data if r["atomic_actions"].get(atomic_action) is not None] actions_data["total"] = [r["duration"] for r in raw_data if not r["error"]] return actions_data
class MainStatsTableTestCase(test.TestCase): @ddt.data( { "info": { "iterations_count": 1, "atomic": costilius.OrderedDict([("foo", {}), ("bar", {})]) }, "data": [ generate_iteration(10.0, False, ("foo", 1.0), ("bar", 2.0)) ], "expected": { "cols": MAIN_STATS_TABLE_COLUMNS, "rows": [ ["foo", 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, "100.0%", 1], ["bar", 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, "100.0%", 1], ["total", 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, "100.0%", 1], ] } }, { "info": {"iterations_count": 2, "atomic": {"foo": {}}}, "data": [ generate_iteration(10.0, True, ("foo", 1.0)), generate_iteration(10.0, True, ("foo", 2.0)) ], "expected": { "cols": MAIN_STATS_TABLE_COLUMNS, "rows": [ ["foo", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", 2], ["total", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", 2], ] } }, { "info": {"iterations_count": 2, "atomic": {"foo": {}}}, "data": [ generate_iteration(10.0, False, ("foo", 1.0)), generate_iteration(20.0, True, ("foo", 2.0)) ], "expected": { "cols": MAIN_STATS_TABLE_COLUMNS, "rows": [ ["foo", 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, "50.0%", 2], ["total", 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, "50.0%", 2] ] } }, { "info": { "iterations_count": 4, "atomic": costilius.OrderedDict([("foo", {}), ("bar", {})]) }, "data": [ generate_iteration(10.0, False, ("foo", 1.0), ("bar", 4.0)), generate_iteration(20.0, False, ("foo", 2.0), ("bar", 4.0)), generate_iteration(30.0, False, ("foo", 3.0), ("bar", 4.0)), generate_iteration(40.0, True, ("foo", 4.0), ("bar", 4.0)) ], "expected": { "cols": MAIN_STATS_TABLE_COLUMNS, "rows": [ ["foo", 1.0, 2.0, 2.8, 2.9, 3.0, 2.0, "75.0%", 4], ["bar", 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, "75.0%", 4], ["total", 10.0, 20.0, 28.0, 29.0, 30.0, 20.0, "75.0%", 4] ] } }, { "info": { "iterations_count": 0, "atomic": costilius.OrderedDict() }, "data": [], "expected": { "cols": MAIN_STATS_TABLE_COLUMNS, "rows": [ ["total", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", 0] ] } } ) @ddt.unpack def test_add_iteration_and_render(self, info, data, expected): table = charts.MainStatsTable(info) for el in data: table.add_iteration(el) self.assertEqual(expected, table.render())
def generate_iteration(duration, error, *args): return { "atomic_actions": costilius.OrderedDict(args), "duration": duration, "error": error }
def __init__(self, context=None): self.context = context self._idle_duration = 0 self._atomic_actions = costilius.OrderedDict()
def extend_results(cls, results, serializable=False): """Modify and extend results with aggregated data. This is a workaround method that tries to adapt task results to schema of planned DB refactoring, so this method is expected to be simplified after DB refactoring since all the data should be taken as-is directly from the database. Each scenario results have extra `info' with aggregated data, and iterations data is represented by iterator - this simplifies its future implementation as generator and gives ability to process arbitrary number of iterations with low memory usage. :param results: list of db.sqlalchemy.models.TaskResult :param serializable: bool, whether to convert json non-serializable types (like datetime) to serializable ones :returns: list of dicts, each dict represents scenario results: key - dict, scenario input data sla - list, SLA results iterations - if serializiable, then iterator with iterations data, otherwise a list created_at - if serializiable, then str datetime, otherwise absent updated_at - if serializiable, then str datetime, otherwise absent info: atomic - dict where key is one of atomic action names and value is dict {min_duration: number, max_duration: number} output_names - list of str output values names (if any) iterations_count - int number of iterations iterations_failed - int number of iterations with errors min_duration - float minimum iteration duration max_duration - float maximum iteration duration tstamp_start - float timestamp of the first iteration full_duration - float full scenario duration load_duration - float load scenario duration """ extended = [] for scenario_result in results: scenario = dict(scenario_result) tstamp_start = 0 min_duration = 0 max_duration = 0 iterations_failed = 0 atomic = costilius.OrderedDict() output_names = set() for itr in scenario["data"]["raw"]: for atomic_name, duration in itr["atomic_actions"].items(): duration = duration or 0 if atomic_name not in atomic: atomic[atomic_name] = { "min_duration": duration, "max_duration": duration } elif duration < atomic[atomic_name]["min_duration"]: atomic[atomic_name]["min_duration"] = duration elif duration > atomic[atomic_name]["max_duration"]: atomic[atomic_name]["max_duration"] = duration output_names.update(itr["scenario_output"]["data"].keys()) if not tstamp_start or itr["timestamp"] < tstamp_start: tstamp_start = itr["timestamp"] if itr["error"]: iterations_failed += 1 else: duration = itr["duration"] or 0 if not min_duration or duration < min_duration: min_duration = duration if not max_duration or duration > max_duration: max_duration = duration for k in "created_at", "updated_at": if serializable: # NOTE(amaretskiy): convert datetime to str, # because json.dumps() does not like datetime if scenario[k]: scenario[k] = scenario[k].strftime("%Y-%d-%mT%H:%M:%S") else: del scenario[k] scenario["info"] = { "atomic": atomic, "output_names": list(output_names), "iterations_count": len(scenario["data"]["raw"]), "iterations_failed": iterations_failed, "min_duration": min_duration, "max_duration": max_duration, "tstamp_start": tstamp_start, "full_duration": scenario["data"]["full_duration"], "load_duration": scenario["data"]["load_duration"] } if serializable: scenario["iterations"] = scenario["data"]["raw"] else: scenario["iterations"] = iter(scenario["data"]["raw"]) scenario["sla"] = scenario["data"]["sla"] del scenario["data"] del scenario["task_uuid"] del scenario["id"] extended.append(scenario) return extended
def __init__(self, context=None, admin_clients=None, clients=None): self.context = context self._admin_clients = admin_clients self._clients = clients self._idle_duration = 0 self._atomic_actions = costilius.OrderedDict()
def _init_columns(self): return costilius.OrderedDict([("foo", self.foo), ("bar", self.bar)])