def test_merge_atomic_actions(self):
expected = [("foo", {
"duration": 2,
"count": 1,
"children": collections.OrderedDict()
}),
("bar", {
"duration": 5,
"count": 2,
"children": collections.OrderedDict()
})]
result = atomic.merge_atomic_actions([{
"name": "foo",
"started_at": 4,
"finished_at": 6,
"children": []
}, {
"name": "bar",
"started_at": 6,
"finished_at": 8,
"children": []
}, {
"name": "bar",
"started_at": 8,
"finished_at": 11,
"children": []
}])
result = list(result.items())
self.assertEqual(expected, result)
def add_iteration(self, iteration):
"""Add data of a single iteration."""
data = atomic.merge_atomic_actions(iteration["atomic_actions"])
# NOTE(andreykurilin): the easiest way to identify the last
# atomic is to find the last added key to the OrderedDict. The
# most perfect way is to use reversed, since class OrderedDict
# uses a doubly linked list for the dictionary items and
# implements __reversed__(), what is why such implementation
# gives you O(1) access to the desired element.
if data:
the_last = data[next(reversed(data))]
if iteration["error"] and not the_last.get("failed", False):
# un-wrapped action failed
data["<no-name-action>"] = {"duration": 0, "count": 1,
"failed": True, "children": {}}
total_duration = iteration["duration"] + iteration["idle_duration"]
data["total"] = {"duration": total_duration,
"count": 1,
"failed": bool(iteration["error"]),
"children": collections.OrderedDict(
[("duration", {
"duration": iteration["duration"],
"count": 1,
"failed": bool(iteration["error"]),
"children": []}),
("idle_duration", {
"duration": iteration["idle_duration"],
"count": 1,
"failed": bool(iteration["error"]),
"children": []})
])}
self._add_data(data)
def add_iteration(self, iteration):
"""Add data of a single iteration."""
data = atomic.merge_atomic_actions(iteration["atomic_actions"])
# NOTE(andreykurilin): the easiest way to identify the last
# atomic is to find the last added key to the OrderedDict. The
# most perfect way is to use reversed, since class OrderedDict
# uses a doubly linked list for the dictionary items and
# implements __reversed__(), what is why such implementation
# gives you O(1) access to the desired element.
if data:
the_last = data[next(reversed(data))]
if iteration["error"] and not the_last.get("failed", False):
# un-wrapped action failed
data["<no-name-action>"] = {"duration": 0, "count": 1,
"failed": True, "children": {}}
total_duration = iteration["duration"] + iteration["idle_duration"]
data["total"] = {"duration": total_duration,
"count": 1,
"failed": bool(iteration["error"]),
"children": collections.OrderedDict(
[("duration", {
"duration": iteration["duration"],
"count": 1,
"failed": bool(iteration["error"]),
"children": []}),
("idle_duration", {
"duration": iteration["idle_duration"],
"count": 1,
"failed": bool(iteration["error"]),
"children": []})
])}
self._add_data(data)
def _map_iteration_values(self, iteration):
atomic_actions = atomic.merge_atomic_actions(
iteration["atomic_actions"])
atomics = self._fix_atomic_actions(atomic_actions)
if self._workload["failed_iteration_count"]:
if iteration["error"]:
failed_duration = (iteration["duration"] +
iteration["idle_duration"] -
sum([(a[1] or 0) for a in atomics]))
else:
failed_duration = 0
atomics.append(("failed_duration", failed_duration))
return atomics
def _map_iteration_values(self, iteration):
atomic_actions = atomic.merge_atomic_actions(
iteration["atomic_actions"])
atomics = self._fix_atomic_actions(atomic_actions)
if self._workload["failed_iteration_count"]:
if iteration["error"]:
failed_duration = (
iteration["duration"] + iteration["idle_duration"]
- sum([(a[1] or 0) for a in atomics]))
else:
failed_duration = 0
atomics.append(("failed_duration", failed_duration))
return atomics
def test_merge_atomic_actions(self):
expected = [("foo", {"duration": 2, "count": 1,
"children": collections.OrderedDict()}),
("bar", {"duration": 5, "count": 2,
"children": collections.OrderedDict()}),
("do_something_bad", {
"duration": 1, "count": 1, "failed": True,
"children": collections.OrderedDict()})]
result = atomic.merge_atomic_actions(
[{"name": "foo", "started_at": 4, "finished_at": 6,
"children": []},
{"name": "bar", "started_at": 6, "finished_at": 8,
"children": []},
{"name": "bar", "started_at": 8, "finished_at": 11,
"children": []},
{"name": "do_something_bad", "started_at": 11, "finished_at": 12,
"children": [], "failed": True}
])
result = list(result.items())
self.assertEqual(expected, result)
def add_iteration(self, iteration):
"""Add data of a single iteration."""
data = atomic.merge_atomic_actions(iteration["atomic_actions"])
if iteration["error"]:
# NOTE(andreykurilin): if an iteration fails, it means that the
# last atomic action produced an error.
# NOTE(andreykurilin): It worse to mention that there is a
# uncovered case when the failed item is not wrapped by atomic
# timer, so possibly the last item in atomic actions list can be
# successful. This thing should be fixed in AtomicTimer.
self._mark_the_last_as_an_error(data)
total_duration = iteration["duration"] + iteration["idle_duration"]
data["total"] = {
"duration":
total_duration,
"count":
1,
"error":
iteration["error"],
"children":
collections.OrderedDict([("duration", {
"duration": iteration["duration"],
"count": 1,
"error": iteration["error"],
"children": []
}),
("idle_duration", {
"duration":
iteration["idle_duration"],
"count": 1,
"error": iteration["error"],
"children": []
})])
}
if iteration["error"]:
data["total"]["error"] = True
self._add_data(data)
def _map_iteration_values(self, iteration):
atomic_actions = atomic.merge_atomic_actions(
iteration["atomic_actions"])
return self._fix_atomic_actions(atomic_actions)
def _map_iteration_values(self, iteration):
atomic_actions = atomic.merge_atomic_actions(
iteration["atomic_actions"])
return self._fix_atomic_actions(atomic_actions)
def _detailed(self,
api,
task_id=None,
iterations_data=False,
filters=None):
"""Print detailed information about given task."""
scenarios_filter = []
only_sla_failures = False
for filter in filters or []:
if filter.startswith("scenario="):
filter_value = filter.split("=")[1]
scenarios_filter = filter_value.split(",")
if filter == "sla-failures":
only_sla_failures = True
task = api.task.get(task_id=task_id, detailed=True)
print()
print("-" * 80)
print("Task %(task_id)s: %(status)s" % {
"task_id": task_id,
"status": task["status"]
})
if task["status"] == consts.TaskStatus.CRASHED or task["status"] == (
consts.TaskStatus.VALIDATION_FAILED):
print("-" * 80)
validation = task["validation_result"]
if logging.is_debug():
print(yaml.safe_load(validation["trace"]))
else:
print(validation["etype"])
print(validation["msg"])
print("\nFor more details run:\nrally -d task detailed %s" %
task["uuid"])
return 0
elif task["status"] not in [
consts.TaskStatus.FINISHED, consts.TaskStatus.ABORTED
]:
print("-" * 80)
print("\nThe task %s marked as '%s'. Results "
"available when it is '%s'." %
(task_id, task["status"], consts.TaskStatus.FINISHED))
return 0
for workload in itertools.chain(
*[s["workloads"] for s in task["subtasks"]]):
if scenarios_filter and workload["name"] not in scenarios_filter:
continue
if only_sla_failures and workload["pass_sla"]:
continue
print("-" * 80)
print()
print("test scenario %s" % workload["name"])
print("args position %s" % workload["position"])
print("args values:")
print(
json.dumps(
{
"args": workload["args"],
"runner": workload["runner"],
"contexts": workload["contexts"],
"sla": workload["sla"],
"hooks": [r["config"] for r in workload["hooks"]]
},
indent=2))
print()
duration_stats = workload["statistics"]["durations"]
iterations = []
iterations_headers = ["iteration", "duration"]
iterations_actions = []
output = []
task_errors = []
if iterations_data:
atomic_names = [
a["display_name"] for a in duration_stats["atomics"]
]
for i, atomic_name in enumerate(atomic_names, 1):
action = "%i. %s" % (i, atomic_name)
iterations_headers.append(action)
iterations_actions.append((atomic_name, action))
for idx, itr in enumerate(workload["data"], 1):
if iterations_data:
row = {"iteration": idx, "duration": itr["duration"]}
for name, action in iterations_actions:
atomic_actions = atomic.merge_atomic_actions(
itr["atomic_actions"])
row[action] = atomic_actions.get(name, {}).get(
"duration", 0)
iterations.append(row)
if "output" in itr:
iteration_output = itr["output"]
else:
iteration_output = {"additive": [], "complete": []}
for idx, additive in enumerate(iteration_output["additive"]):
if len(output) <= idx + 1:
output_table = charts.OutputStatsTable(
workload, title=additive["title"])
output.append(output_table)
output[idx].add_iteration(additive["data"])
if itr.get("error"):
task_errors.append(TaskCommands._format_task_error(itr))
self._print_task_errors(task_id, task_errors)
cols = charts.MainStatsTable.columns
formatters = {
"Action": lambda x: x["display_name"],
"Min (sec)": lambda x: x["data"]["min"],
"Median (sec)": lambda x: x["data"]["median"],
"90%ile (sec)": lambda x: x["data"]["90%ile"],
"95%ile (sec)": lambda x: x["data"]["95%ile"],
"Max (sec)": lambda x: x["data"]["max"],
"Avg (sec)": lambda x: x["data"]["avg"],
"Success": lambda x: x["data"]["success"],
"Count": lambda x: x["data"]["iteration_count"]
}
rows = []
def make_flat(r, depth=0):
if depth > 0:
r["display_name"] = (" %s> %s" %
("-" * depth, r["display_name"]))
rows.append(r)
for children in r["children"]:
make_flat(children, depth + 1)
for row in itertools.chain(duration_stats["atomics"],
[duration_stats["total"]]):
make_flat(row)
cliutils.print_list(rows,
fields=cols,
formatters=formatters,
normalize_field_names=True,
table_label="Response Times (sec)",
sortby_index=None)
print()
if iterations_data:
formatters = dict(
zip(iterations_headers[1:], [
cliutils.pretty_float_formatter(col, 3)
for col in iterations_headers[1:]
]))
cliutils.print_list(iterations,
fields=iterations_headers,
table_label="Atomics per iteration",
formatters=formatters)
print()
if output:
cols = charts.OutputStatsTable.columns
float_cols = cols[1:7]
formatters = dict(
zip(float_cols, [
cliutils.pretty_float_formatter(col, 3)
for col in float_cols
]))
for out in output:
data = out.render()
rows = [dict(zip(cols, r)) for r in data["data"]["rows"]]
if rows:
# NOTE(amaretskiy): print title explicitly because
# prettytable fails if title length is too long
print(data["title"])
cliutils.print_list(rows,
fields=cols,
formatters=formatters)
print()
print("Load duration: %s" %
strutils.format_float_to_str(workload["load_duration"]))
print("Full duration: %s" %
strutils.format_float_to_str(workload["full_duration"]))
print("\nHINTS:")
print("* To plot HTML graphics with this data, run:")
print("\trally task report %s --out output.html\n" % task["uuid"])
print("* To generate a JUnit report, run:")
print("\trally task export %s --type junit --to output.xml\n" %
task["uuid"])
print("* To get raw JSON output of task results, run:")
print("\trally task report %s --json --out output.json\n" %
task["uuid"])
if not task["pass_sla"]:
print("At least one workload did not pass SLA criteria.\n")
return 1
def _detailed(self, api, task_id=None, iterations_data=False):
"""Print detailed information about given task."""
task = api.task.get(task_id=task_id, detailed=True)
print()
print("-" * 80)
print("Task %(task_id)s: %(status)s"
% {"task_id": task_id, "status": task["status"]})
if task["status"] == consts.TaskStatus.CRASHED or task["status"] == (
consts.TaskStatus.VALIDATION_FAILED):
print("-" * 80)
validation = task["validation_result"]
if logging.is_debug():
print(yaml.safe_load(validation["trace"]))
else:
print(validation["etype"])
print(validation["msg"])
print("\nFor more details run:\nrally -d task detailed %s"
% task["uuid"])
return 0
elif task["status"] not in [consts.TaskStatus.FINISHED,
consts.TaskStatus.ABORTED]:
print("-" * 80)
print("\nThe task %s marked as '%s'. Results "
"available when it is '%s'."
% (task_id, task["status"], consts.TaskStatus.FINISHED))
return 0
for workload in itertools.chain(
*[s["workloads"] for s in task["subtasks"]]):
print("-" * 80)
print()
print("test scenario %s" % workload["name"])
print("args position %s" % workload["position"])
print("args values:")
print(json.dumps(
{"args": workload["args"],
"runner": workload["runner"],
"contexts": workload["contexts"],
"sla": workload["sla"],
"hooks": [r["config"] for r in workload["hooks"]]},
indent=2))
print()
duration_stats = workload["statistics"]["durations"]
iterations = []
iterations_headers = ["iteration", "duration"]
iterations_actions = []
output = []
task_errors = []
if iterations_data:
atomic_names = [a["display_name"]
for a in duration_stats["atomics"]]
for i, atomic_name in enumerate(atomic_names, 1):
action = "%i. %s" % (i, atomic_name)
iterations_headers.append(action)
iterations_actions.append((atomic_name, action))
for idx, itr in enumerate(workload["data"], 1):
if iterations_data:
row = {"iteration": idx, "duration": itr["duration"]}
for name, action in iterations_actions:
atomic_actions = atomic.merge_atomic_actions(
itr["atomic_actions"])
row[action] = atomic_actions.get(name, {}).get(
"duration", 0)
iterations.append(row)
if "output" in itr:
iteration_output = itr["output"]
else:
iteration_output = {"additive": [], "complete": []}
for idx, additive in enumerate(iteration_output["additive"]):
if len(output) <= idx + 1:
output_table = charts.OutputStatsTable(
workload, title=additive["title"])
output.append(output_table)
output[idx].add_iteration(additive["data"])
if itr.get("error"):
task_errors.append(TaskCommands._format_task_error(itr))
self._print_task_errors(task_id, task_errors)
cols = charts.MainStatsTable.columns
formatters = {
"Action": lambda x: x["display_name"],
"Min (sec)": lambda x: x["data"]["min"],
"Median (sec)": lambda x: x["data"]["median"],
"90%ile (sec)": lambda x: x["data"]["90%ile"],
"95%ile (sec)": lambda x: x["data"]["95%ile"],
"Max (sec)": lambda x: x["data"]["max"],
"Avg (sec)": lambda x: x["data"]["avg"],
"Success": lambda x: x["data"]["success"],
"Count": lambda x: x["data"]["iteration_count"]
}
rows = []
def make_flat(r, depth=0):
if depth > 0:
r["display_name"] = (" %s> %s" % ("-" * depth,
r["display_name"]))
rows.append(r)
for children in r["children"]:
make_flat(children, depth + 1)
for row in itertools.chain(duration_stats["atomics"],
[duration_stats["total"]]):
make_flat(row)
cliutils.print_list(rows,
fields=cols,
formatters=formatters,
normalize_field_names=True,
table_label="Response Times (sec)",
sortby_index=None)
print()
if iterations_data:
formatters = dict(zip(iterations_headers[1:],
[cliutils.pretty_float_formatter(col, 3)
for col in iterations_headers[1:]]))
cliutils.print_list(iterations,
fields=iterations_headers,
table_label="Atomics per iteration",
formatters=formatters)
print()
if output:
cols = charts.OutputStatsTable.columns
float_cols = cols[1:7]
formatters = dict(zip(float_cols,
[cliutils.pretty_float_formatter(col, 3)
for col in float_cols]))
for out in output:
data = out.render()
rows = [dict(zip(cols, r)) for r in data["data"]["rows"]]
if rows:
# NOTE(amaretskiy): print title explicitly because
# prettytable fails if title length is too long
print(data["title"])
cliutils.print_list(rows, fields=cols,
formatters=formatters)
print()
print("Load duration: %s"
% strutils.format_float_to_str(workload["load_duration"]))
print("Full duration: %s"
% strutils.format_float_to_str(workload["full_duration"]))
print("\nHINTS:")
print("* To plot HTML graphics with this data, run:")
print("\trally task report %s --out output.html\n" % task["uuid"])
print("* To generate a JUnit report, run:")
print("\trally task export %s --type junit --to output.xml\n" %
task["uuid"])
print("* To get raw JSON output of task results, run:")
print("\trally task report %s --json --out output.json\n" %
task["uuid"])
if not task["pass_sla"]:
print("At least one workload did not pass SLA criteria.\n")
return 1
