def test_optional_action_timer_decorator(self, mock_time):
class TestAtomicTimer(atomic.ActionTimerMixin):
@atomic.optional_action_timer("some")
def some_func(self, a, b):
return a + b
@atomic.optional_action_timer("some",
argument_name="foo",
default=False)
def other_func(self, a, b):
return a + b
inst = TestAtomicTimer()
self.assertEqual(5, inst.some_func(2, 3))
self.assertEqual(costilius.OrderedDict({"some": 2}),
inst.atomic_actions())
inst = TestAtomicTimer()
self.assertEqual(5, inst.some_func(2, 3, atomic_action=False))
self.assertEqual(costilius.OrderedDict(), inst.atomic_actions())
inst = TestAtomicTimer()
self.assertEqual(5, inst.other_func(2, 3))
self.assertEqual(costilius.OrderedDict(), inst.atomic_actions())
inst = TestAtomicTimer()
self.assertEqual(5, inst.other_func(2, 3, foo=True))
self.assertEqual(costilius.OrderedDict({"some": 2}),
inst.atomic_actions())
def test_add_iteration_and_render(self):
self.assertRaises(TypeError, charts.Table, self.bench_info)
table = self.Table(self.bench_info)
self.assertEqual(costilius.OrderedDict(), table.render())
[table.add_iteration({"a": i, "b": 43 - i}) for i in range(1, 43)]
self.assertEqual(
costilius.OrderedDict([
("value_a",
costilius.OrderedDict([("foo", table.foo),
("bar", table.bar)])),
("value_b",
costilius.OrderedDict([("foo", table.foo),
("bar", table.bar)]))
]), table.render())
def test_add_iteration_and_render(self):
chart = charts.AtomicAvgChart({"iterations_count": 3,
"atomic": {"foo": {}, "bar": {}}})
self.assertIsInstance(chart, charts.AvgChart)
[chart.add_iteration({"atomic_actions": costilius.OrderedDict(a)})
for a in ([("foo", 2), ("bar", 5)], [("foo", 4)], [("bar", 7)])]
self.assertEqual([("bar", 4.0), ("foo", 2.0)], sorted(chart.render()))
def test_add_iteration_and_render(self):
chart = charts.AtomicHistogramChart(
{"iterations_count": 3,
"atomic": costilius.OrderedDict(
[("foo", {"min_duration": 1.6, "max_duration": 2.8}),
("bar", {"min_duration": 3.1, "max_duration": 5.5})])})
self.assertIsInstance(chart, charts.HistogramChart)
[chart.add_iteration({"atomic_actions": a})
for a in ({"foo": 1.6, "bar": 3.1}, {"foo": 2.8}, {"bar": 5.5})]
expected = [
[{"disabled": 0, "key": "foo", "view": "Square Root Choice",
"values": [{"x": 2.2, "y": 2}, {"x": 2.8, "y": 1}]},
{"disabled": 0, "key": "foo", "view": "Sturges Formula",
"values": [{"x": 2.0, "y": 2}, {"x": 2.4, "y": 0},
{"x": 2.8, "y": 1}]},
{"disabled": 0, "key": "foo", "view": "Rice Rule",
"values": [{"x": 2.0, "y": 2}, {"x": 2.4, "y": 0},
{"x": 2.8, "y": 1}]}],
[{"disabled": 1, "key": "bar", "view": "Square Root Choice",
"values": [{"x": 4.3, "y": 2}, {"x": 5.5, "y": 1}]},
{"disabled": 1, "key": "bar", "view": "Sturges Formula",
"values": [{"x": 3.9, "y": 2}, {"x": 4.7, "y": 0},
{"x": 5.5, "y": 1}]},
{"disabled": 1, "key": "bar", "view": "Rice Rule",
"values": [{"x": 3.9, "y": 2}, {"x": 4.7, "y": 0},
{"x": 5.5, "y": 1}]}]]
self.assertEqual(expected, chart.render())
def test_add_iteration_and_render(self):
table = charts.MainStatsTable({
"iterations_count": 42,
"atomic": {
"foo": {},
"bar": {}
}
})
[
table.add_iteration({
"atomic_actions":
costilius.OrderedDict([("foo", i), ("bar", 43 - 1)]),
"duration":
i,
"error":
i % 40
}) for i in range(1, 43)
]
expected_rows = [[
"foo", 1.0, 21.5, 38.5, 40.5, 42.0, 21.5, "100.0%", 42.0
], ["bar", 42.0, 42.0, 42.0, 42.0, 42.0, 42.0, "100.0%",
42.0], ["total", 0.0, 0.0, 0.0, 0.0, 40.0, 0.952, "100.0%", 42.0]]
self.assertEqual({
"cols": self.columns,
"rows": expected_rows
}, table.render())
def test_action_timer_decorator(self, mock_time):
class Some(atomic.ActionTimerMixin):
@atomic.action_timer("some")
def some_func(self, a, b):
return a + b
inst = Some()
self.assertEqual(5, inst.some_func(2, 3))
self.assertEqual(costilius.OrderedDict({"some": 2}),
inst.atomic_actions())
def _init_columns(self):
return costilius.OrderedDict(
(("Min (sec)", streaming.MinComputation()),
("Median (sec)", streaming.PercentileComputation(50)),
("90%ile (sec)", streaming.PercentileComputation(90)),
("95%ile (sec)", streaming.PercentileComputation(95)),
("Max (sec)", streaming.MaxComputation()),
("Avg (sec)", streaming.MeanComputation()),
("Success", streaming.ProgressComputation(self.base_size)),
("Count", streaming.IncrementComputation())))
def test_action_timer_context(self, mock_time):
inst = atomic.ActionTimerMixin()
with atomic.ActionTimer(inst, "test"):
with atomic.ActionTimer(inst, "test"):
with atomic.ActionTimer(inst, "some"):
pass
expected = [("test", 20), ("test (2)", 12), ("some", 4)]
self.assertEqual(costilius.OrderedDict(expected),
inst.atomic_actions())
def __init__(self, benchmark_info, zipped_size=1000):
"""Setup initial values.
:param benchmark_info: dict, generalized info about iterations.
The most important value is `iterations_count'
that should have int value of total data size
:param zipped_size: int maximum number of points on scale
"""
self._data = costilius.OrderedDict() # Container for results
self._benchmark_info = benchmark_info
self.base_size = benchmark_info.get("iterations_count", 0)
self.zipped_size = zipped_size
def test_action_timer_decorator_with_exception(self, mock_time):
class TestException(Exception):
pass
class TestTimer(atomic.ActionTimerMixin):
@atomic.action_timer("test")
def some_func(self):
raise TestException("test")
inst = TestTimer()
self.assertRaises(TestException, inst.some_func)
self.assertEqual(costilius.OrderedDict({"test": 2}),
inst.atomic_actions())
def test_action_timer_context_with_exception(self, mock_time):
inst = atomic.ActionTimerMixin()
class TestException(Exception):
pass
try:
with atomic.ActionTimer(inst, "test"):
raise TestException("test")
except TestException:
pass
expected = [("test", 2)]
self.assertEqual(costilius.OrderedDict(expected),
inst.atomic_actions())
def get_atomic_actions_data(raw_data):
"""Retrieve detailed (by atomic actions & total runtime) benchmark data.
:parameter raw_data: list of raw records (scenario runner output)
:returns: dictionary containing atomic action + total duration lists
for all atomic action keys
"""
atomic_actions = []
for row in raw_data:
# find first non-error result to get atomic actions names
if not row["error"] and "atomic_actions" in row:
atomic_actions = row["atomic_actions"].keys()
break
actions_data = costilius.OrderedDict()
for atomic_action in atomic_actions:
actions_data[atomic_action] = [
r["atomic_actions"][atomic_action]
for r in raw_data
if r["atomic_actions"].get(atomic_action) is not None]
actions_data["total"] = [r["duration"] for r in raw_data if not r["error"]]
return actions_data
class MainStatsTableTestCase(test.TestCase):
@ddt.data(
{
"info": {
"iterations_count": 1,
"atomic": costilius.OrderedDict([("foo", {}), ("bar", {})])
},
"data": [
generate_iteration(10.0, False, ("foo", 1.0), ("bar", 2.0))
],
"expected": {
"cols": MAIN_STATS_TABLE_COLUMNS,
"rows": [
["foo", 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, "100.0%", 1],
["bar", 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, "100.0%", 1],
["total", 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, "100.0%", 1],
]
}
},
{
"info": {"iterations_count": 2, "atomic": {"foo": {}}},
"data": [
generate_iteration(10.0, True, ("foo", 1.0)),
generate_iteration(10.0, True, ("foo", 2.0))
],
"expected": {
"cols": MAIN_STATS_TABLE_COLUMNS,
"rows": [
["foo", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a",
2],
["total", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a",
2],
]
}
},
{
"info": {"iterations_count": 2, "atomic": {"foo": {}}},
"data": [
generate_iteration(10.0, False, ("foo", 1.0)),
generate_iteration(20.0, True, ("foo", 2.0))
],
"expected": {
"cols": MAIN_STATS_TABLE_COLUMNS,
"rows": [
["foo", 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, "50.0%", 2],
["total", 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, "50.0%", 2]
]
}
},
{
"info": {
"iterations_count": 4,
"atomic": costilius.OrderedDict([("foo", {}), ("bar", {})])
},
"data": [
generate_iteration(10.0, False, ("foo", 1.0), ("bar", 4.0)),
generate_iteration(20.0, False, ("foo", 2.0), ("bar", 4.0)),
generate_iteration(30.0, False, ("foo", 3.0), ("bar", 4.0)),
generate_iteration(40.0, True, ("foo", 4.0), ("bar", 4.0))
],
"expected": {
"cols": MAIN_STATS_TABLE_COLUMNS,
"rows": [
["foo", 1.0, 2.0, 2.8, 2.9, 3.0, 2.0, "75.0%", 4],
["bar", 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, "75.0%", 4],
["total", 10.0, 20.0, 28.0, 29.0, 30.0, 20.0, "75.0%", 4]
]
}
},
{
"info": {
"iterations_count": 0,
"atomic": costilius.OrderedDict()
},
"data": [],
"expected": {
"cols": MAIN_STATS_TABLE_COLUMNS,
"rows": [
["total", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a", "n/a",
0]
]
}
}
)
@ddt.unpack
def test_add_iteration_and_render(self, info, data, expected):
table = charts.MainStatsTable(info)
for el in data:
table.add_iteration(el)
self.assertEqual(expected, table.render())
def generate_iteration(duration, error, *args):
return {
"atomic_actions": costilius.OrderedDict(args),
"duration": duration,
"error": error
}
def extend_results(cls, results, serializable=False):
"""Modify and extend results with aggregated data.
This is a workaround method that tries to adapt task results
to schema of planned DB refactoring, so this method is expected
to be simplified after DB refactoring since all the data should
be taken as-is directly from the database.
Each scenario results have extra `info' with aggregated data,
and iterations data is represented by iterator - this simplifies
its future implementation as generator and gives ability to process
arbitrary number of iterations with low memory usage.
:param results: list of db.sqlalchemy.models.TaskResult
:param serializable: bool, whether to convert json non-serializable
types (like datetime) to serializable ones
:returns: list of dicts, each dict represents scenario results:
key - dict, scenario input data
sla - list, SLA results
iterations - if serializiable, then iterator with
iterations data, otherwise a list
created_at - if serializiable, then str datetime,
otherwise absent
updated_at - if serializiable, then str datetime,
otherwise absent
info:
atomic - dict where key is one of atomic action names
and value is dict {min_duration: number,
max_duration: number}
output_names - list of str output values names (if any)
iterations_count - int number of iterations
iterations_failed - int number of iterations with errors
min_duration - float minimum iteration duration
max_duration - float maximum iteration duration
tstamp_start - float timestamp of the first iteration
full_duration - float full scenario duration
load_duration - float load scenario duration
"""
extended = []
for scenario_result in results:
scenario = dict(scenario_result)
tstamp_start = 0
min_duration = 0
max_duration = 0
iterations_failed = 0
atomic = costilius.OrderedDict()
output_names = set()
for itr in scenario["data"]["raw"]:
for atomic_name, duration in itr["atomic_actions"].items():
duration = duration or 0
if atomic_name not in atomic:
atomic[atomic_name] = {
"min_duration": duration,
"max_duration": duration
}
elif duration < atomic[atomic_name]["min_duration"]:
atomic[atomic_name]["min_duration"] = duration
elif duration > atomic[atomic_name]["max_duration"]:
atomic[atomic_name]["max_duration"] = duration
output_names.update(itr["scenario_output"]["data"].keys())
if not tstamp_start or itr["timestamp"] < tstamp_start:
tstamp_start = itr["timestamp"]
if itr["error"]:
iterations_failed += 1
else:
duration = itr["duration"] or 0
if not min_duration or duration < min_duration:
min_duration = duration
if not max_duration or duration > max_duration:
max_duration = duration
for k in "created_at", "updated_at":
if serializable:
# NOTE(amaretskiy): convert datetime to str,
# because json.dumps() does not like datetime
if scenario[k]:
scenario[k] = scenario[k].strftime("%Y-%d-%mT%H:%M:%S")
else:
del scenario[k]
scenario["info"] = {
"atomic": atomic,
"output_names": list(output_names),
"iterations_count": len(scenario["data"]["raw"]),
"iterations_failed": iterations_failed,
"min_duration": min_duration,
"max_duration": max_duration,
"tstamp_start": tstamp_start,
"full_duration": scenario["data"]["full_duration"],
"load_duration": scenario["data"]["load_duration"]
}
if serializable:
scenario["iterations"] = scenario["data"]["raw"]
else:
scenario["iterations"] = iter(scenario["data"]["raw"])
scenario["sla"] = scenario["data"]["sla"]
del scenario["data"]
del scenario["task_uuid"]
del scenario["id"]
extended.append(scenario)
return extended
def __init__(self):
self._atomic_actions = costilius.OrderedDict()
def __init__(self, context=None):
self.context = context
self._idle_duration = 0
self._atomic_actions = costilius.OrderedDict()
def _init_columns(self):
return costilius.OrderedDict([("foo", self.foo),
("bar", self.bar)])
def __init__(self, context=None, admin_clients=None, clients=None):
self.context = context
self._admin_clients = admin_clients
self._clients = clients
self._idle_duration = 0
self._atomic_actions = costilius.OrderedDict()
