def _process_row(self, sa):
points_ins, min_v, max_v, avg, success, count = sa
# process percentiles
points = points_ins.result()
# clear the space on the disk
points_ins.reset()
# sort data once
points.sort()
p50ile = utils.percentile(points, 0.5, ignore_sorting=True)
p90ile = utils.percentile(points, 0.9, ignore_sorting=True)
p95ile = utils.percentile(points, 0.95, ignore_sorting=True)
# process and round values
count = count.result()
has_result = bool(count)
min_v = self._round(min_v, has_result)
max_v = self._round(max_v, has_result)
avg = self._round(avg, has_result)
p50ile = self._round(p50ile, has_result)
p90ile = self._round(p90ile, has_result)
p95ile = self._round(p95ile, has_result)
success = "%.1f%%" % (success.result() * 100) if has_result else "n/a"
return min_v, max_v, avg, p50ile, p90ile, p95ile, success, count
def _process_row(self, name, sa):
points_ins, count, min_v, max_v, avg = sa
# process percentiles
points = points_ins.result()
# clear the space on the disk
points_ins.reset()
# sort data once
points.sort()
p50ile = utils.percentile(points, 0.5, ignore_sorting=True)
p90ile = utils.percentile(points, 0.9, ignore_sorting=True)
p95ile = utils.percentile(points, 0.95, ignore_sorting=True)
# process and round values
count = count.result()
has_result = bool(count)
min_v = self._round(min_v, has_result)
max_v = self._round(max_v, has_result)
avg = self._round(avg, has_result)
p50ile = self._round(p50ile, has_result)
p90ile = self._round(p90ile, has_result)
p95ile = self._round(p95ile, has_result)
return [name, min_v, p50ile, p90ile, p95ile, max_v, avg, count]
def _get_atomic_action_durations(result):
raw = result.get("result", [])
actions_data = utils.get_atomic_actions_data(raw)
table = []
total = []
for action in actions_data:
durations = actions_data[action]
if durations:
data = [action,
round(min(durations), 3),
round(utils.median(durations), 3),
round(utils.percentile(durations, 0.90), 3),
round(utils.percentile(durations, 0.95), 3),
round(max(durations), 3),
round(utils.mean(durations), 3),
"%.1f%%" % (len(durations) * 100.0 / len(raw)),
len(raw)]
else:
data = [action, None, None, None, None, None, None, 0, len(raw)]
# Save 'total' - it must be appended last
if action == "total":
total = data
continue
table.append(data)
if total:
table.append(total)
return table
def _print_ssrs_result(result):
raw = result["data"]["raw"]
# NOTE(hughsaunders): ssrs=scenario specific results
ssrs = []
for result in raw:
data = result["scenario_output"].get("data")
if data:
ssrs.append(data)
if ssrs:
keys = set()
for ssr in ssrs:
keys.update(ssr.keys())
headers = [
"key", "min", "median", "90%ile", "95%ile", "max", "avg"
]
float_cols = [
"min", "median", "90%ile", "95%ile", "max", "avg"
]
formatters = dict(
zip(float_cols, [
cliutils.pretty_float_formatter(col, 3)
for col in float_cols
]))
table_rows = []
for key in keys:
values = [float(ssr[key]) for ssr in ssrs if key in ssr]
if values:
row = [
str(key),
round(min(values), 3),
round(utils.median(values), 3),
round(utils.percentile(values, 0.90), 3),
round(utils.percentile(values, 0.95), 3),
round(max(values), 3),
round(utils.mean(values), 3)
]
else:
row = [str(key)] + ["n/a"] * 6
table_rows.append(rutils.Struct(**dict(zip(headers, row))))
print("\nScenario Specific Results\n")
cliutils.print_list(table_rows,
fields=headers,
formatters=formatters,
table_label="Response Times (sec)")
for result in raw:
errors = result["scenario_output"].get("errors")
if errors:
print(errors)
def _print_ssrs_result(result):
raw = result["data"]["raw"]
# NOTE(hughsaunders): ssrs=scenario specific results
ssrs = []
for result in raw:
data = result["scenario_output"].get("data")
if data:
ssrs.append(data)
if ssrs:
keys = set()
for ssr in ssrs:
keys.update(ssr.keys())
headers = ["key", "min", "median",
"90%ile", "95%ile", "max",
"avg"]
float_cols = ["min", "median", "90%ile",
"95%ile", "max", "avg"]
formatters = dict(zip(float_cols,
[cliutils.pretty_float_formatter(col, 3)
for col in float_cols]))
table_rows = []
for key in keys:
values = [float(ssr[key]) for ssr in ssrs if key in ssr]
if values:
row = [str(key),
round(min(values), 3),
round(utils.median(values), 3),
round(utils.percentile(values, 0.90), 3),
round(utils.percentile(values, 0.95), 3),
round(max(values), 3),
round(utils.mean(values), 3)]
else:
row = [str(key)] + ["n/a"] * 6
table_rows.append(rutils.Struct(**dict(zip(headers,
row))))
print("\nScenario Specific Results\n")
cliutils.print_list(table_rows,
fields=headers,
formatters=formatters,
table_label="Response Times (sec)")
for result in raw:
errors = result["scenario_output"].get("errors")
if errors:
print(errors)
def _print_summrized_result(result):
raw = result["data"]["raw"]
table_cols = [
"action", "min", "median", "90%ile", "95%ile", "max", "avg",
"success", "count"
]
float_cols = ["min", "median", "90%ile", "95%ile", "max", "avg"]
formatters = dict(
zip(float_cols, [
cliutils.pretty_float_formatter(col, 3)
for col in float_cols
]))
table_rows = []
actions_data = utils.get_atomic_actions_data(raw)
for action in actions_data:
durations = actions_data[action]
if durations:
data = [
action,
round(min(durations), 3),
round(utils.median(durations), 3),
round(utils.percentile(durations, 0.90), 3),
round(utils.percentile(durations, 0.95), 3),
round(max(durations), 3),
round(utils.mean(durations), 3),
"%.1f%%" % (len(durations) * 100.0 / len(raw)),
len(raw)
]
else:
data = [
action, None, None, None, None, None, None, "0.0%",
len(raw)
]
table_rows.append(rutils.Struct(**dict(zip(table_cols, data))))
cliutils.print_list(table_rows,
fields=table_cols,
formatters=formatters,
table_label="Response Times (sec)",
sortby_index=None)
def _print_summrized_result(result):
raw = result["data"]["raw"]
table_cols = ["action", "min", "median",
"90%ile", "95%ile", "max",
"avg", "success", "count"]
float_cols = ["min", "median",
"90%ile", "95%ile", "max",
"avg"]
formatters = dict(zip(float_cols,
[cliutils.pretty_float_formatter(col, 3)
for col in float_cols]))
table_rows = []
actions_data = utils.get_atomic_actions_data(raw)
for action in actions_data:
durations = actions_data[action]
if durations:
data = [action,
round(min(durations), 3),
round(utils.median(durations), 3),
round(utils.percentile(durations, 0.90), 3),
round(utils.percentile(durations, 0.95), 3),
round(max(durations), 3),
round(utils.mean(durations), 3),
"%.1f%%" % (len(durations) * 100.0 / len(raw)),
len(raw)]
else:
data = [action, None, None, None, None, None, None,
"0.0%", len(raw)]
table_rows.append(rutils.Struct(**dict(zip(table_cols,
data))))
cliutils.print_list(table_rows, fields=table_cols,
formatters=formatters,
table_label="Response Times (sec)",
sortby_index=None)
def _print_ssrs_result(result):
raw = result["data"]["raw"]
# NOTE(hughsaunders): ssrs=scenario specific results
ssrs = []
for itr in raw:
if "output" not in itr:
itr["output"] = {"additive": [], "complete": []}
# NOTE(amaretskiy): "scenario_output" is supported
# for backward compatibility
if ("scenario_output" in itr
and itr["scenario_output"]["data"]):
itr["output"]["additive"].append(
{"data": itr["scenario_output"]["data"].items(),
"title": "Scenario output",
"description": "",
"chart_plugin": "StackedArea"})
del itr["scenario_output"]
for idx, additive in enumerate(itr["output"]["additive"]):
try:
for key, value in additive["data"]:
ssrs[idx]["data"][key].append(value)
except IndexError:
data = {}
keys = []
for key, value in additive["data"]:
if key not in data:
data[key] = []
keys.append(key)
data[key].append(value)
ssrs.append({"title": additive["title"],
"keys": keys,
"data": data})
if not ssrs:
return
print("\nScenario Specific Results\n")
headers = ["key", "min", "median", "90%ile", "95%ile",
"max", "avg"]
float_cols = ["min", "median", "90%ile", "95%ile", "max", "avg"]
formatters = dict(zip(float_cols,
[cliutils.pretty_float_formatter(col, 3)
for col in float_cols]))
for ssr in ssrs:
rows = []
for key in ssr["keys"]:
values = ssr["data"][key]
if values:
row = [str(key),
round(min(values), 3),
round(utils.median(values), 3),
round(utils.percentile(values, 0.90), 3),
round(utils.percentile(values, 0.95), 3),
round(max(values), 3),
round(utils.mean(values), 3)]
else:
row = [str(key)] + ["n/a"] * 6
rows.append(rutils.Struct(**dict(zip(headers, row))))
cliutils.print_list(rows,
fields=headers,
formatters=formatters,
table_label=ssr["title"])
print()
def detailed(self, task_id=None, iterations_data=False):
"""Display results table.
:param task_id: Task uuid
:param iterations_data: print detailed results for each iteration
Prints detailed information of task.
"""
def _print_iterations_data(raw_data):
headers = ["iteration", "full duration"]
float_cols = ["full duration"]
atomic_actions = []
for row in raw_data:
# find first non-error result to get atomic actions names
if not row["error"] and "atomic_actions" in row:
atomic_actions = row["atomic_actions"].keys()
for row in raw_data:
if row["atomic_actions"]:
for (c, a) in enumerate(atomic_actions, 1):
action = "%(no)i. %(action)s" % {"no": c, "action": a}
headers.append(action)
float_cols.append(action)
break
table_rows = []
formatters = dict(zip(float_cols,
[cliutils.pretty_float_formatter(col, 3)
for col in float_cols]))
for (c, r) in enumerate(raw_data, 1):
dlist = [c]
dlist.append(r["duration"])
if r["atomic_actions"]:
for action in atomic_actions:
dlist.append(r["atomic_actions"].get(action) or 0)
table_rows.append(rutils.Struct(**dict(zip(headers, dlist))))
cliutils.print_list(table_rows,
fields=headers,
formatters=formatters)
print()
task = db.task_get_detailed(task_id)
if task is None:
print("The task %s can not be found" % task_id)
return(1)
print()
print("-" * 80)
print(_("Task %(task_id)s: %(status)s")
% {"task_id": task_id, "status": task["status"]})
if task["status"] == consts.TaskStatus.FAILED:
print("-" * 80)
verification = yaml.safe_load(task["verification_log"])
if not logging.is_debug():
print(verification[0])
print(verification[1])
print()
print(_("For more details run:\nrally -vd task detailed %s")
% task["uuid"])
else:
print(yaml.safe_load(verification[2]))
return
for result in task["results"]:
key = result["key"]
print("-" * 80)
print()
print("test scenario %s" % key["name"])
print("args position %s" % key["pos"])
print("args values:")
print(json.dumps(key["kw"], indent=2))
raw = result["data"]["raw"]
table_cols = ["action", "min", "median",
"90%ile", "95%ile", "max",
"avg", "success", "count"]
float_cols = ["min", "median",
"90%ile", "95%ile", "max",
"avg"]
formatters = dict(zip(float_cols,
[cliutils.pretty_float_formatter(col, 3)
for col in float_cols]))
table_rows = []
actions_data = utils.get_atomic_actions_data(raw)
for action in actions_data:
durations = actions_data[action]
if durations:
data = [action,
round(min(durations), 3),
round(utils.median(durations), 3),
round(utils.percentile(durations, 0.90), 3),
round(utils.percentile(durations, 0.95), 3),
round(max(durations), 3),
round(utils.mean(durations), 3),
"%.1f%%" % (len(durations) * 100.0 / len(raw)),
len(raw)]
else:
data = [action, None, None, None, None, None, None,
"0.0%", len(raw)]
table_rows.append(rutils.Struct(**dict(zip(table_cols, data))))
cliutils.print_list(table_rows, fields=table_cols,
formatters=formatters,
table_label="Response Times (sec)",
sortby_index=None)
if iterations_data:
_print_iterations_data(raw)
print(_("Load duration: %s") % result["data"]["load_duration"])
print(_("Full duration: %s") % result["data"]["full_duration"])
# NOTE(hughsaunders): ssrs=scenario specific results
ssrs = []
for result in raw:
data = result["scenario_output"].get("data")
if data:
ssrs.append(data)
if ssrs:
keys = set()
for ssr in ssrs:
keys.update(ssr.keys())
headers = ["key", "min", "median",
"90%ile", "95%ile", "max",
"avg"]
float_cols = ["min", "median", "90%ile",
"95%ile", "max", "avg"]
formatters = dict(zip(float_cols,
[cliutils.pretty_float_formatter(col, 3)
for col in float_cols]))
table_rows = []
for key in keys:
values = [float(ssr[key]) for ssr in ssrs if key in ssr]
if values:
row = [str(key),
round(min(values), 3),
round(utils.median(values), 3),
round(utils.percentile(values, 0.90), 3),
round(utils.percentile(values, 0.95), 3),
round(max(values), 3),
round(utils.mean(values), 3)]
else:
row = [str(key)] + ["n/a"] * 6
table_rows.append(rutils.Struct(**dict(zip(headers, row))))
print("\nScenario Specific Results\n")
cliutils.print_list(table_rows,
fields=headers,
formatters=formatters,
table_label="Response Times (sec)")
for result in raw:
errors = result["scenario_output"].get("errors")
if errors:
print(errors)
print()
print("HINTS:")
print(_("* To plot HTML graphics with this data, run:"))
print("\trally task report %s --out output.html" % task["uuid"])
print()
print(_("* To generate a JUnit report, run:"))
print("\trally task report %s --junit --out output.xml" %
task["uuid"])
print()
print(_("* To get raw JSON output of task results, run:"))
print("\trally task results %s\n" % task["uuid"])
def _print_ssrs_result(result):
raw = result["data"]["raw"]
# NOTE(hughsaunders): ssrs=scenario specific results
ssrs = []
for itr in raw:
if "output" not in itr:
itr["output"] = {"additive": [], "complete": []}
# NOTE(amaretskiy): "scenario_output" is supported
# for backward compatibility
if ("scenario_output" in itr
and itr["scenario_output"]["data"]):
itr["output"]["additive"].append({
"data":
itr["scenario_output"]["data"].items(),
"title":
"Scenario output",
"description":
"",
"chart_plugin":
"StackedArea"
})
del itr["scenario_output"]
for idx, additive in enumerate(itr["output"]["additive"]):
try:
for key, value in additive["data"]:
ssrs[idx]["data"][key].append(value)
except IndexError:
data = {}
keys = []
for key, value in additive["data"]:
if key not in data:
data[key] = []
keys.append(key)
data[key].append(value)
ssrs.append({
"title": additive["title"],
"keys": keys,
"data": data
})
if not ssrs:
return
print("\nScenario Specific Results\n")
headers = [
"key", "min", "median", "90%ile", "95%ile", "max", "avg"
]
float_cols = ["min", "median", "90%ile", "95%ile", "max", "avg"]
formatters = dict(
zip(float_cols, [
cliutils.pretty_float_formatter(col, 3)
for col in float_cols
]))
for ssr in ssrs:
rows = []
for key in ssr["keys"]:
values = ssr["data"][key]
if values:
row = [
str(key),
round(min(values), 3),
round(utils.median(values), 3),
round(utils.percentile(values, 0.90), 3),
round(utils.percentile(values, 0.95), 3),
round(max(values), 3),
round(utils.mean(values), 3)
]
else:
row = [str(key)] + ["n/a"] * 6
rows.append(rutils.Struct(**dict(zip(headers, row))))
cliutils.print_list(rows,
fields=headers,
formatters=formatters,
table_label=ssr["title"])
print()
def test_result_empty(self):
self.assertIsNone(
utils.percentile([], percent=0.50, ignore_sorting=True))
def test_percentile_equal(self):
lst = list(range(1, 101))
result = utils.percentile(lst, 1)
self.assertEqual(result, 100)
def test_percentile_value_none(self):
result = utils.percentile(None, 0.1)
self.assertIsNone(result)
def test_percentile_equal(self):
lst = list(range(1, 101))
result = utils.percentile(lst, 1)
self.assertEqual(result, 100)
def test_percentile_value_none(self):
result = utils.percentile(None, 0.1)
self.assertIsNone(result)
def result(self):
results = list(
map(lambda x: x[1], self._graph_zipper.get_zipped_graph()))
if results:
return utils.percentile(results, self._percent)
return None
def result(self):
results = list(
map(lambda x: x[1], self._graph_zipper.get_zipped_graph()))
if results:
return utils.percentile(results, self._percent)
return None
def detailed(self, task_id=None, iterations_data=False):
"""Display results table.
:param task_id: Task uuid
:param iterations_data: print detailed results for each iteration
Prints detailed information of task.
"""
def _print_iterations_data(raw_data):
headers = ["iteration", "full duration"]
float_cols = ["full duration"]
atomic_actions = []
for row in raw_data:
# find first non-error result to get atomic actions names
if not row["error"] and "atomic_actions" in row:
atomic_actions = row["atomic_actions"].keys()
for row in raw_data:
if row["atomic_actions"]:
for (c, a) in enumerate(atomic_actions, 1):
action = "%(no)i. %(action)s" % {"no": c, "action": a}
headers.append(action)
float_cols.append(action)
break
table_rows = []
formatters = dict(
zip(float_cols, [
cliutils.pretty_float_formatter(col, 3)
for col in float_cols
]))
for (c, r) in enumerate(raw_data, 1):
dlist = [c]
dlist.append(r["duration"])
if r["atomic_actions"]:
for action in atomic_actions:
dlist.append(r["atomic_actions"].get(action) or 0)
table_rows.append(rutils.Struct(**dict(zip(headers, dlist))))
cliutils.print_list(table_rows,
fields=headers,
formatters=formatters)
print()
task = db.task_get_detailed(task_id)
if task is None:
print("The task %s can not be found" % task_id)
return (1)
print()
print("-" * 80)
print(
_("Task %(task_id)s: %(status)s") % {
"task_id": task_id,
"status": task["status"]
})
if task["status"] == consts.TaskStatus.FAILED:
print("-" * 80)
verification = yaml.safe_load(task["verification_log"])
if not logging.is_debug():
print(verification[0])
print(verification[1])
print()
print(
_("For more details run:\nrally -vd task detailed %s") %
task["uuid"])
else:
print(yaml.safe_load(verification[2]))
return
for result in task["results"]:
key = result["key"]
print("-" * 80)
print()
print("test scenario %s" % key["name"])
print("args position %s" % key["pos"])
print("args values:")
print(json.dumps(key["kw"], indent=2))
raw = result["data"]["raw"]
table_cols = [
"action", "min", "median", "90%ile", "95%ile", "max", "avg",
"success", "count"
]
float_cols = ["min", "median", "90%ile", "95%ile", "max", "avg"]
formatters = dict(
zip(float_cols, [
cliutils.pretty_float_formatter(col, 3)
for col in float_cols
]))
table_rows = []
actions_data = utils.get_atomic_actions_data(raw)
for action in actions_data:
durations = actions_data[action]
if durations:
data = [
action,
round(min(durations), 3),
round(utils.median(durations), 3),
round(utils.percentile(durations, 0.90), 3),
round(utils.percentile(durations, 0.95), 3),
round(max(durations), 3),
round(utils.mean(durations), 3),
"%.1f%%" % (len(durations) * 100.0 / len(raw)),
len(raw)
]
else:
data = [
action, None, None, None, None, None, None, "0.0%",
len(raw)
]
table_rows.append(rutils.Struct(**dict(zip(table_cols, data))))
cliutils.print_list(table_rows,
fields=table_cols,
formatters=formatters,
table_label="Response Times (sec)",
sortby_index=None)
if iterations_data:
_print_iterations_data(raw)
print(_("Load duration: %s") % result["data"]["load_duration"])
print(_("Full duration: %s") % result["data"]["full_duration"])
# NOTE(hughsaunders): ssrs=scenario specific results
ssrs = []
for result in raw:
data = result["scenario_output"].get("data")
if data:
ssrs.append(data)
if ssrs:
keys = set()
for ssr in ssrs:
keys.update(ssr.keys())
headers = [
"key", "min", "median", "90%ile", "95%ile", "max", "avg"
]
float_cols = [
"min", "median", "90%ile", "95%ile", "max", "avg"
]
formatters = dict(
zip(float_cols, [
cliutils.pretty_float_formatter(col, 3)
for col in float_cols
]))
table_rows = []
for key in keys:
values = [float(ssr[key]) for ssr in ssrs if key in ssr]
if values:
row = [
str(key),
round(min(values), 3),
round(utils.median(values), 3),
round(utils.percentile(values, 0.90), 3),
round(utils.percentile(values, 0.95), 3),
round(max(values), 3),
round(utils.mean(values), 3)
]
else:
row = [str(key)] + ["n/a"] * 6
table_rows.append(rutils.Struct(**dict(zip(headers, row))))
print("\nScenario Specific Results\n")
cliutils.print_list(table_rows,
fields=headers,
formatters=formatters,
table_label="Response Times (sec)")
for result in raw:
errors = result["scenario_output"].get("errors")
if errors:
print(errors)
print()
print("HINTS:")
print(_("* To plot HTML graphics with this data, run:"))
print("\trally task report %s --out output.html" % task["uuid"])
print()
print(_("* To generate a JUnit report, run:"))
print("\trally task report %s --junit --out output.xml" % task["uuid"])
print()
print(_("* To get raw JSON output of task results, run:"))
print("\trally task results %s\n" % task["uuid"])
def result(self):
results = list(
map(lambda x: x[1], self._graph_zipper.get_zipped_graph()))
if not results:
raise ValueError("No values have been processed")
return utils.percentile(results, self._percent)
def test_add_and_result(self, percent, stream, expected):
stream = copy.copy(getattr(self, stream))
self.assertEqual(expected,
utils.percentile(percent=percent, points=stream))