def detailed(self, task_id): """Get detailed information about task :param task_id: Task uuid Prints detailed infomration of task. """ task = db.task_get_detailed(task_id) print() print("=" * 80) print(_("Task %(task_id)s is %(status)s.") % {'task_id': task_id, 'status': task['status']}) for result in task["results"]: key = result["key"] print("-" * 80) print() print("test scenario %s" % key["name"]) print("args position %s" % key["pos"]) print("args values:") pprint.pprint(key["kw"]) raw = result["data"]["raw"] times = map(lambda x: x['time'], filter(lambda r: not r['error'], raw)) table = prettytable.PrettyTable(["max", "avg", "min", "ratio"]) table.add_row([max(times), sum(times) / len(times), min(times), float(len(times)) / len(raw)]) print(table)
def test_task_get_detailed(self): task1 = self._create_task() key = {'name': 'atata'} data = {'a': 'b', 'c': 'd'} db.task_result_create(task1['uuid'], key, data) task1_full = db.task_get_detailed(task1['uuid']) results = task1_full["results"] self.assertEqual(len(results), 1) self.assertEqual(results[0]["key"], key) self.assertEqual(results[0]["data"], data)
def test_task_get_detailed(self): task1 = self._create_task() key = {"name": "atata"} data = {"a": "b", "c": "d"} db.task_result_create(task1["uuid"], key, data) task1_full = db.task_get_detailed(task1["uuid"]) results = task1_full["results"] self.assertEqual(len(results), 1) self.assertEqual(results[0]["key"], key) self.assertEqual(results[0]["data"], data)
def sla_check(self, task_id=None, tojson=False): """Check if task was succeded according to SLA. :param task_id: Task uuid. :returns: Number of failed criteria. """ task = db.task_get_detailed(task_id) failed_criteria = 0 rows = [] for row in base_sla.SLA.check_all(task): failed_criteria += 0 if row['success'] else 1 rows.append(row if tojson else rutils.Struct(**row)) if tojson: print(json.dumps(rows)) else: common_cliutils.print_list(rows, ('benchmark', 'pos', 'criterion', 'success')) return failed_criteria
def sla_check(self, task_id=None, tojson=False): """Check if task was succeded according to SLA. :param task_id: Task uuid. :returns: Number of failed criteria. """ task = db.task_get_detailed(task_id) failed_criteria = 0 rows = [] for row in base_sla.SLA.check_all(task): failed_criteria += 0 if row['success'] else 1 rows.append(row if tojson else rutils.Struct(**row)) if tojson: print(json.dumps(rows)) else: common_cliutils.print_list(rows, ('benchmark', 'pos', 'criterion', 'success', 'detail')) return failed_criteria
def detailed(self, task_id=None, iterations_data=False): """Display results table. :param task_id: Task uuid :param iterations_data: print detailed results for each iteration Prints detailed information of task. """ def _print_iterations_data(raw_data): headers = ["iteration", "full duration"] float_cols = ["full duration"] atomic_actions = [] for row in raw_data: # find first non-error result to get atomic actions names if not row["error"] and "atomic_actions" in row: atomic_actions = row["atomic_actions"].keys() for row in raw_data: if row["atomic_actions"]: for (c, a) in enumerate(atomic_actions, 1): action = "%(no)i. %(action)s" % {"no": c, "action": a} headers.append(action) float_cols.append(action) break table_rows = [] formatters = dict( zip(float_cols, [ cliutils.pretty_float_formatter(col, 3) for col in float_cols ])) for (c, r) in enumerate(raw_data, 1): dlist = [c] dlist.append(r["duration"]) if r["atomic_actions"]: for action in atomic_actions: dlist.append(r["atomic_actions"].get(action) or 0) cliutils.print_list(table_rows, fields=headers, formatters=formatters) print() task = db.task_get_detailed(task_id) if task is None: print("The task %s can not be found" % task_id) return (1) print() print("-" * 80) print( _("Task %(task_id)s: %(status)s") % { "task_id": task_id, "status": task["status"] }) if task["status"] == consts.TaskStatus.FAILED: print("-" * 80) verification = yaml.safe_load(task["verification_log"]) if not logging.is_debug(): print(verification[0]) print(verification[1]) print() print( _("For more details run:\nrally -vd task detailed %s") % task["uuid"]) else: print(yaml.safe_load(verification[2])) return for result in task["results"]: key = result["key"] print("-" * 80) print() print("test scenario %s" % key["name"]) print("args position %s" % key["pos"]) print("args values:") print(json.dumps(key["kw"], indent=2)) raw = result["data"]["raw"] table_cols = [ "action", "min (sec)", "avg (sec)", "max (sec)", "90 percentile", "95 percentile", "success", "count" ] float_cols = [ "min (sec)", "avg (sec)", "max (sec)", "90 percentile", "95 percentile" ] formatters = dict( zip(float_cols, [ cliutils.pretty_float_formatter(col, 3) for col in float_cols ])) table_rows = [] actions_data = utils.get_atomic_actions_data(raw) for action in actions_data: durations = actions_data[action] if durations: data = [ action, min(durations), utils.mean(durations), max(durations), utils.percentile(durations, 0.90), utils.percentile(durations, 0.95), "%.1f%%" % (len(durations) * 100.0 / len(raw)), len(raw) ] else: data = [ action, None, None, None, None, None, "0.0%", len(raw) ] table_rows.append(rutils.Struct(**dict(zip(table_cols, data)))) cliutils.print_list(table_rows, fields=table_cols, formatters=formatters) if iterations_data: _print_iterations_data(raw) print(_("Load duration: %s") % result["data"]["load_duration"]) print(_("Full duration: %s") % result["data"]["full_duration"]) # NOTE(hughsaunders): ssrs=scenario specific results ssrs = [] for result in raw: data = result["scenario_output"].get("data") if data: ssrs.append(data) if ssrs: keys = set() for ssr in ssrs: keys.update(ssr.keys()) headers = [ "key", "max", "avg", "min", "90 pecentile", "95 pecentile" ] float_cols = [ "max", "avg", "min", "90 pecentile", "95 pecentile" ] formatters = dict( zip(float_cols, [ cliutils.pretty_float_formatter(col, 3) for col in float_cols ])) table_rows = [] for key in keys: values = [float(ssr[key]) for ssr in ssrs if key in ssr] if values: row = [ str(key), max(values), utils.mean(values), min(values), utils.percentile(values, 0.90), utils.percentile(values, 0.95) ] else: row = [str(key)] + ["n/a"] * 5 table_rows.append(rutils.Struct(**dict(zip(headers, row)))) print("\nScenario Specific Results\n") cliutils.print_list(table_rows, fields=headers, formatters=formatters) for result in raw: errors = result["scenario_output"].get("errors") if errors: print(errors) print() print("HINTS:") print(_("* To plot HTML graphics with this data, run:")) print("\trally task report %s --out output.html" % task["uuid"]) print() print(_("* To get raw JSON output of task results, run:")) print("\trally task results %s\n" % task["uuid"])
def detailed(self, task_id=None, no_aggregation=False): """Get detailed information about task :param task_id: Task uuid :param no_aggregation: do not aggregate atomic operations Prints detailed information of task. """ def _print_atomic_actions_no_aggregation(raw): headers = ['iteration', "full duration"] for i in range(0, len(raw)): if raw[i]['atomic_actions']: for (c, a) in enumerate(raw[i]['atomic_actions'], 1): action = str(c) + "-" + a['action'] headers.append(action) break atomic_action_table = prettytable.PrettyTable(headers) for (c, r) in enumerate(raw, 1): dlist = [c] d = [] if r['atomic_actions']: for l in r['atomic_actions']: d.append(l['duration']) dlist.append(sum(d)) dlist = dlist + d atomic_action_table.add_row(dlist) else: atomic_action_table.add_row(dlist + ["N/A" for i in range(1, len(headers))]) print(atomic_action_table) print() def _print_atomic_actions_aggregation(raw): aduration_merged = [] for r in raw: if 'atomic_actions' in r: for a in r['atomic_actions']: aduration_merged.append(a) durations_by_action = collections.defaultdict(list) for at in aduration_merged: durations_by_action[at['action']].append(at['duration']) if durations_by_action: atomic_action_table = prettytable.PrettyTable( ['action', 'count', 'max (sec)', 'avg (sec)', 'min (sec)', '90 percentile', '95 percentile']) for k, v in durations_by_action.iteritems(): atomic_action_table.add_row([k, len(v), max(v), sum(v) / len(v), min(v), percentile(v, 0.90), percentile(v, 0.95)]) print(atomic_action_table) print() def _print_atomic_actions(raw): if no_aggregation: _print_atomic_actions_no_aggregation(raw) else: _print_atomic_actions_aggregation(raw) if task_id == "last": task = db.task_get_detailed_last() task_id = task.uuid else: task = db.task_get_detailed(task_id) if task is None: print("The task %s can not be found" % task_id) return print() print("=" * 80) print(_("Task %(task_id)s is %(status)s.") % {"task_id": task_id, "status": task["status"]}) if task["failed"]: print("-" * 80) verification = yaml.safe_load(task["verification_log"]) if not cfg.CONF.debug: print(verification[0]) print(verification[1]) print() print(_("For more details run:\nrally -vd task detailed %s") % task["uuid"]) else: print(yaml.safe_load(verification[2])) return for result in task["results"]: key = result["key"] print("-" * 80) print() print("test scenario %s" % key["name"]) print("args position %s" % key["pos"]) print("args values:") pprint.pprint(key["kw"]) _print_atomic_actions(result["data"]["raw"]) raw = result["data"]["raw"] durations = map(lambda x: x['duration'], filter(lambda r: not r['error'], raw)) table = prettytable.PrettyTable(["max (sec)", "avg (sec)", "min (sec)", "90 pecentile", "95 percentile", "success/total", "total times"]) if durations: table.add_row([max(durations), sum(durations) / len(durations), min(durations), percentile(durations, 0.90), percentile(durations, 0.95), float(len(durations)) / len(raw), len(raw)]) else: table.add_row(['n/a', 'n/a', 'n/a', 'n/a', 'n/a', 0, len(raw)]) print(table) # NOTE(hughsaunders): ssrs=scenario specific results ssrs = [] for result in raw: try: ssrs.append(result['scenario_output']['data']) except (KeyError, TypeError): # No SSRs in this result pass if ssrs: keys = set() for ssr in ssrs: keys.update(ssr.keys()) ssr_table = prettytable.PrettyTable(["Key", "max", "avg", "min", "90 pecentile", "95 pecentile"]) for key in keys: values = [float(ssr[key]) for ssr in ssrs if key in ssr] if values: row = [str(key), max(values), sum(values) / len(values), min(values), percentile(values, 0.90), percentile(values, 0.95)] else: row = [str(key)] + ['n/a'] * 5 ssr_table.add_row(row) print("\nScenario Specific Results\n") print(ssr_table) for result in raw: if result['scenario_output']['errors']: print(result['scenario_output']['errors']) print() print("HINTS:") print(_("* To plot HTML graphics with this data, run:")) print("\trally task plot2html %s --out output.html" % task["uuid"]) print() print(_("* To get raw JSON output of task results, run:")) print("\trally task results %s\n" % task["uuid"])
def detailed(self, task_id=None, iterations_data=False): """Display results table. :param task_id: Task uuid :param iterations_data: print detailed results for each iteration Prints detailed information of task. """ def _print_iterations_data(raw_data): headers = ["iteration", "full duration"] float_cols = ["full duration"] atomic_actions = [] for row in raw_data: # find first non-error result to get atomic actions names if not row["error"] and "atomic_actions" in row: atomic_actions = row["atomic_actions"].keys() for row in raw_data: if row["atomic_actions"]: for (c, a) in enumerate(atomic_actions, 1): action = "%(no)i. %(action)s" % {"no": c, "action": a} headers.append(action) float_cols.append(action) break table_rows = [] formatters = dict(zip(float_cols, [cliutils.pretty_float_formatter(col, 3) for col in float_cols])) for (c, r) in enumerate(raw_data, 1): dlist = [c] dlist.append(r["duration"]) if r["atomic_actions"]: for action in atomic_actions: dlist.append(r["atomic_actions"].get(action) or 0) cliutils.print_list(table_rows, fields=headers, formatters=formatters) print() task = db.task_get_detailed(task_id) if task is None: print("The task %s can not be found" % task_id) return(1) print() print("-" * 80) print(_("Task %(task_id)s: %(status)s") % {"task_id": task_id, "status": task["status"]}) if task["status"] == consts.TaskStatus.FAILED: print("-" * 80) verification = yaml.safe_load(task["verification_log"]) if not logging.is_debug(): print(verification[0]) print(verification[1]) print() print(_("For more details run:\nrally -vd task detailed %s") % task["uuid"]) else: print(yaml.safe_load(verification[2])) return for result in task["results"]: key = result["key"] print("-" * 80) print() print("test scenario %s" % key["name"]) print("args position %s" % key["pos"]) print("args values:") print(json.dumps(key["kw"], indent=2)) raw = result["data"]["raw"] table_cols = ["action", "min (sec)", "avg (sec)", "max (sec)", "90 percentile", "95 percentile", "success", "count"] float_cols = ["min (sec)", "avg (sec)", "max (sec)", "90 percentile", "95 percentile"] formatters = dict(zip(float_cols, [cliutils.pretty_float_formatter(col, 3) for col in float_cols])) table_rows = [] actions_data = utils.get_atomic_actions_data(raw) for action in actions_data: durations = actions_data[action] if durations: data = [action, min(durations), utils.mean(durations), max(durations), utils.percentile(durations, 0.90), utils.percentile(durations, 0.95), "%.1f%%" % (len(durations) * 100.0 / len(raw)), len(raw)] else: data = [action, None, None, None, None, None, "0.0%", len(raw)] table_rows.append(rutils.Struct(**dict(zip(table_cols, data)))) cliutils.print_list(table_rows, fields=table_cols, formatters=formatters) if iterations_data: _print_iterations_data(raw) print(_("Load duration: %s") % result["data"]["load_duration"]) print(_("Full duration: %s") % result["data"]["full_duration"]) # NOTE(hughsaunders): ssrs=scenario specific results ssrs = [] for result in raw: data = result["scenario_output"].get("data") if data: ssrs.append(data) if ssrs: keys = set() for ssr in ssrs: keys.update(ssr.keys()) headers = ["key", "max", "avg", "min", "90 pecentile", "95 pecentile"] float_cols = ["max", "avg", "min", "90 pecentile", "95 pecentile"] formatters = dict(zip(float_cols, [cliutils.pretty_float_formatter(col, 3) for col in float_cols])) table_rows = [] for key in keys: values = [float(ssr[key]) for ssr in ssrs if key in ssr] if values: row = [str(key), max(values), utils.mean(values), min(values), utils.percentile(values, 0.90), utils.percentile(values, 0.95)] else: row = [str(key)] + ["n/a"] * 5 table_rows.append(rutils.Struct(**dict(zip(headers, row)))) print("\nScenario Specific Results\n") cliutils.print_list(table_rows, fields=headers, formatters=formatters) for result in raw: errors = result["scenario_output"].get("errors") if errors: print(errors) print() print("HINTS:") print(_("* To plot HTML graphics with this data, run:")) print("\trally task report %s --out output.html" % task["uuid"]) print() print(_("* To get raw JSON output of task results, run:")) print("\trally task results %s\n" % task["uuid"])
def detailed(self, task_id=None, iterations_data=False): """Get detailed information about task :param task_id: Task uuid :param iterations_data: print detailed results for each iteration Prints detailed information of task. """ def _print_iterations_data(raw): headers = ['iteration', "full duration"] float_cols = ['full duration'] for i in range(0, len(raw)): if raw[i]['atomic_actions']: for (c, a) in enumerate(raw[i]['atomic_actions'], 1): action = str(c) + "-" + a['action'] headers.append(action) float_cols.append(action) break table_rows = [] formatters = dict(zip(float_cols, [cliutils.pretty_float_formatter(col, 3) for col in float_cols])) for (c, r) in enumerate(raw, 1): dlist = [c] d = [] if r['atomic_actions']: for l in r['atomic_actions']: d.append(l['duration']) dlist.append(sum(d)) dlist = dlist + d table_rows.append(rutils.Struct(**dict(zip(headers, dlist)))) else: data = dlist + ["N/A" for i in range(1, len(headers))] table_rows.append(rutils.Struct(**dict(zip(headers, data)))) common_cliutils.print_list(table_rows, fields=headers, formatters=formatters) print() def _get_atomic_action_durations(raw): atomic_actions_names = [] for r in raw: if 'atomic_actions' in r: for a in r['atomic_actions']: atomic_actions_names.append(a["action"]) break result = {} for atomic_action in atomic_actions_names: result[atomic_action] = utils.get_durations( raw, lambda r: next(a["duration"] for a in r["atomic_actions"] if a["action"] == atomic_action), lambda r: any((a["action"] == atomic_action) for a in r["atomic_actions"])) return result if task_id == "last": task = db.task_get_detailed_last() task_id = task.uuid else: task = db.task_get_detailed(task_id) if task is None: print("The task %s can not be found" % task_id) return(1) print() print("=" * 80) print(_("Task %(task_id)s is %(status)s.") % {"task_id": task_id, "status": task["status"]}) if task["failed"]: print("-" * 80) verification = yaml.safe_load(task["verification_log"]) if not cfg.CONF.debug: print(verification[0]) print(verification[1]) print() print(_("For more details run:\nrally -vd task detailed %s") % task["uuid"]) else: print(yaml.safe_load(verification[2])) return for result in task["results"]: key = result["key"] print("-" * 80) print() print("test scenario %s" % key["name"]) print("args position %s" % key["pos"]) print("args values:") pprint.pprint(key["kw"]) raw = result["data"]["raw"] table_cols = ["action", "min (sec)", "avg (sec)", "max (sec)", "90 percentile", "95 percentile", "success", "count"] float_cols = ["min (sec)", "avg (sec)", "max (sec)", "90 percentile", "95 percentile"] formatters = dict(zip(float_cols, [cliutils.pretty_float_formatter(col, 3) for col in float_cols])) table_rows = [] action_durations = _get_atomic_action_durations(raw) actions_list = action_durations.keys() action_durations["total"] = utils.get_durations( raw, lambda x: x["duration"], lambda r: not r["error"]) actions_list.append("total") for action in actions_list: durations = action_durations[action] if durations: data = [action, min(durations), utils.mean(durations), max(durations), utils.percentile(durations, 0.90), utils.percentile(durations, 0.95), "%.1f%%" % (len(durations) * 100.0 / len(raw)), len(raw)] else: data = [action, None, None, None, None, None, 0, len(raw)] table_rows.append(rutils.Struct(**dict(zip(table_cols, data)))) common_cliutils.print_list(table_rows, fields=table_cols, formatters=formatters) if iterations_data: _print_iterations_data(raw) # NOTE(hughsaunders): ssrs=scenario specific results ssrs = [] for result in raw: data = result["scenario_output"].get("data") if data: ssrs.append(data) if ssrs: keys = set() for ssr in ssrs: keys.update(ssr.keys()) headers = ["key", "max", "avg", "min", "90 pecentile", "95 pecentile"] float_cols = ["max", "avg", "min", "90 pecentile", "95 pecentile"] formatters = dict(zip(float_cols, [cliutils.pretty_float_formatter(col, 3) for col in float_cols])) table_rows = [] for key in keys: values = [float(ssr[key]) for ssr in ssrs if key in ssr] if values: row = [str(key), max(values), utils.mean(values), min(values), utils.percentile(values, 0.90), utils.percentile(values, 0.95)] else: row = [str(key)] + ['n/a'] * 5 table_rows.append(rutils.Struct(**dict(zip(headers, row)))) print("\nScenario Specific Results\n") common_cliutils.print_list(table_rows, fields=headers, formatters=formatters) for result in raw: errors = result["scenario_output"].get("errors") if errors: print(errors) print() print("HINTS:") print(_("* To plot HTML graphics with this data, run:")) print("\trally task plot2html %s --out output.html" % task["uuid"]) print() print(_("* To get raw JSON output of task results, run:")) print("\trally task results %s\n" % task["uuid"])
def detailed(self, task_id=None, no_aggregation=False): """Get detailed information about task :param task_id: Task uuid :param no_aggregation: do not aggregate atomic operations Prints detailed information of task. """ def _print_atomic_actions_no_aggregation(raw): headers = ['iteration', "full duration"] for i in range(0, len(raw)): if raw[i]['atomic_actions']: for (c, a) in enumerate(raw[i]['atomic_actions'], 1): action = str(c) + "-" + a['action'] headers.append(action) break atomic_action_table = prettytable.PrettyTable(headers) for (c, r) in enumerate(raw, 1): dlist = [c] d = [] if r['atomic_actions']: for l in r['atomic_actions']: d.append(l['duration']) dlist.append(sum(d)) dlist = dlist + d atomic_action_table.add_row(dlist) else: atomic_action_table.add_row( dlist + ["N/A" for i in range(1, len(headers))]) print(atomic_action_table) print() def _print_atomic_actions_aggregation(raw): aduration_merged = [] for r in raw: if 'atomic_actions' in r: for a in r['atomic_actions']: aduration_merged.append(a) durations_by_action = collections.defaultdict(list) for at in aduration_merged: durations_by_action[at['action']].append(at['duration']) if durations_by_action: atomic_action_table = prettytable.PrettyTable([ 'action', 'count', 'max (sec)', 'avg (sec)', 'min (sec)', '90 percentile', '95 percentile' ]) for k, v in durations_by_action.iteritems(): atomic_action_table.add_row([ k, len(v), max(v), utils.mean(v), min(v), utils.percentile(v, 0.90), utils.percentile(v, 0.95) ]) print(atomic_action_table) print() def _print_atomic_actions(raw): if no_aggregation: _print_atomic_actions_no_aggregation(raw) else: _print_atomic_actions_aggregation(raw) if task_id == "last": task = db.task_get_detailed_last() task_id = task.uuid else: task = db.task_get_detailed(task_id) if task is None: print("The task %s can not be found" % task_id) return print() print("=" * 80) print( _("Task %(task_id)s is %(status)s.") % { "task_id": task_id, "status": task["status"] }) if task["failed"]: print("-" * 80) verification = yaml.safe_load(task["verification_log"]) if not cfg.CONF.debug: print(verification[0]) print(verification[1]) print() print( _("For more details run:\nrally -vd task detailed %s") % task["uuid"]) else: print(yaml.safe_load(verification[2])) return for result in task["results"]: key = result["key"] print("-" * 80) print() print("test scenario %s" % key["name"]) print("args position %s" % key["pos"]) print("args values:") pprint.pprint(key["kw"]) _print_atomic_actions(result["data"]["raw"]) raw = result["data"]["raw"] durations = utils.get_durations(raw, lambda x: x['duration'], lambda r: not r['error']) table = prettytable.PrettyTable([ "max (sec)", "avg (sec)", "min (sec)", "90 pecentile", "95 percentile", "success/total", "total times" ]) if durations: table.add_row([ max(durations), utils.mean(durations), min(durations), utils.percentile(durations, 0.90), utils.percentile(durations, 0.95), float(len(durations)) / len(raw), len(raw) ]) else: table.add_row(['n/a', 'n/a', 'n/a', 'n/a', 'n/a', 0, len(raw)]) print(table) # NOTE(hughsaunders): ssrs=scenario specific results ssrs = [] for result in raw: try: ssrs.append(result['scenario_output']['data']) except (KeyError, TypeError): # No SSRs in this result pass if ssrs: keys = set() for ssr in ssrs: keys.update(ssr.keys()) ssr_table = prettytable.PrettyTable([ "Key", "max", "avg", "min", "90 pecentile", "95 pecentile" ]) for key in keys: values = [float(ssr[key]) for ssr in ssrs if key in ssr] if values: row = [ str(key), max(values), utils.mean(values), min(values), utils.percentile(values, 0.90), utils.percentile(values, 0.95) ] else: row = [str(key)] + ['n/a'] * 5 ssr_table.add_row(row) print("\nScenario Specific Results\n") print(ssr_table) for result in raw: if result['scenario_output']['errors']: print(result['scenario_output']['errors']) print() print("HINTS:") print(_("* To plot HTML graphics with this data, run:")) print("\trally task plot2html %s --out output.html" % task["uuid"]) print() print(_("* To get raw JSON output of task results, run:")) print("\trally task results %s\n" % task["uuid"])
def detailed(self, task_id=None, iterations_data=False): """Get detailed information about task :param task_id: Task uuid :param iterations_data: print detailed results for each iteration Prints detailed information of task. """ def _print_iterations_data(raw): headers = ['iteration', "full duration"] float_cols = ['full duration'] for i in range(0, len(raw)): if raw[i]['atomic_actions']: for (c, a) in enumerate(raw[i]['atomic_actions'], 1): action = str(c) + "-" + a['action'] headers.append(action) float_cols.append(action) break table_rows = [] formatters = dict(zip(float_cols, [cliutils.pretty_float_formatter(col, 3) for col in float_cols])) for (c, r) in enumerate(raw, 1): dlist = [c] d = [] if r['atomic_actions']: for l in r['atomic_actions']: d.append(l['duration']) dlist.append(sum(d)) dlist = dlist + d table_rows.append(rutils.Struct(**dict(zip(headers, dlist)))) else: data = dlist + ["N/A" for i in range(1, len(headers))] table_rows.append(rutils.Struct(**dict(zip(headers, data)))) common_cliutils.print_list(table_rows, fields=headers, formatters=formatters) print() def _get_atomic_action_durations(raw): atomic_actions_names = [] for r in raw: if 'atomic_actions' in r: for a in r['atomic_actions']: atomic_actions_names.append(a["action"]) break result = {} for atomic_action in atomic_actions_names: result[atomic_action] = utils.get_durations( raw, lambda r: next(a["duration"] for a in r["atomic_actions"] if a["action"] == atomic_action), lambda r: any((a["action"] == atomic_action) for a in r["atomic_actions"])) return result if task_id == "last": task = db.task_get_detailed_last() task_id = task.uuid else: task = db.task_get_detailed(task_id) if task is None: print("The task %s can not be found" % task_id) return(1) print() print("=" * 80) print(_("Task %(task_id)s is %(status)s.") % {"task_id": task_id, "status": task["status"]}) if task["failed"]: print("-" * 80) verification = yaml.safe_load(task["verification_log"]) if not cfg.CONF.debug: print(verification[0]) print(verification[1]) print() print(_("For more details run:\nrally -vd task detailed %s") % task["uuid"]) else: print(yaml.safe_load(verification[2])) return for result in task["results"]: key = result["key"] print("-" * 80) print() print("test scenario %s" % key["name"]) print("args position %s" % key["pos"]) print("args values:") pprint.pprint(key["kw"]) scenario_time = result["data"]["scenario_duration"] raw = result["data"]["raw"] table_cols = ["action", "min (sec)", "avg (sec)", "max (sec)", "90 percentile", "95 percentile", "success", "count"] float_cols = ["min (sec)", "avg (sec)", "max (sec)", "90 percentile", "95 percentile"] formatters = dict(zip(float_cols, [cliutils.pretty_float_formatter(col, 3) for col in float_cols])) table_rows = [] action_durations = _get_atomic_action_durations(raw) actions_list = action_durations.keys() action_durations["total"] = utils.get_durations( raw, lambda x: x["duration"], lambda r: not r["error"]) actions_list.append("total") for action in actions_list: durations = action_durations[action] if durations: data = [action, min(durations), utils.mean(durations), max(durations), utils.percentile(durations, 0.90), utils.percentile(durations, 0.95), "%.1f%%" % (len(durations) * 100.0 / len(raw)), len(raw)] else: data = [action, None, None, None, None, None, 0, len(raw)] table_rows.append(rutils.Struct(**dict(zip(table_cols, data)))) common_cliutils.print_list(table_rows, fields=table_cols, formatters=formatters) if iterations_data: _print_iterations_data(raw) print(_("Whole scenario time without context preparation: "), scenario_time) # NOTE(hughsaunders): ssrs=scenario specific results ssrs = [] for result in raw: data = result["scenario_output"].get("data") if data: ssrs.append(data) if ssrs: keys = set() for ssr in ssrs: keys.update(ssr.keys()) headers = ["key", "max", "avg", "min", "90 pecentile", "95 pecentile"] float_cols = ["max", "avg", "min", "90 pecentile", "95 pecentile"] formatters = dict(zip(float_cols, [cliutils.pretty_float_formatter(col, 3) for col in float_cols])) table_rows = [] for key in keys: values = [float(ssr[key]) for ssr in ssrs if key in ssr] if values: row = [str(key), max(values), utils.mean(values), min(values), utils.percentile(values, 0.90), utils.percentile(values, 0.95)] else: row = [str(key)] + ['n/a'] * 5 table_rows.append(rutils.Struct(**dict(zip(headers, row)))) print("\nScenario Specific Results\n") common_cliutils.print_list(table_rows, fields=headers, formatters=formatters) for result in raw: errors = result["scenario_output"].get("errors") if errors: print(errors) print() print("HINTS:") print(_("* To plot HTML graphics with this data, run:")) print("\trally task plot2html %s --out output.html" % task["uuid"]) print() print(_("* To get raw JSON output of task results, run:")) print("\trally task results %s\n" % task["uuid"])
def aggregated_plot(task_id, aggregated_field): """Draws an aggregated figure of benchmark runtimes in a separate window. The resulting figure has the aggregated field values on the X axis and the benchmark runtimes (in seconds) on the Y axis. For each benchmark run, minimum, maximum and average runtime values will be drawn, thus resulting in three plots on the figure. :param task_id: ID of the task to draw the plot for :param aggregated_field: Field from the test config to aggregate the data on. This can be e.g. "active_users", "times" etc. """ task = db.task_get_detailed(task_id) task["results"].sort(key=lambda res: res["key"]["name"]) results_by_benchmark = itertools.groupby(task["results"], lambda res: res["key"]["name"]) for benchmark_name, data in results_by_benchmark: data_dict = {} for result in data: if aggregated_field not in result["key"]["kw"]["config"]: raise exceptions.NoSuchConfigField(name=aggregated_field) raw = result["data"]["raw"] times = map(lambda x: x["time"], filter(lambda r: not r["error"], raw)) aggr_field_val = result["key"]["kw"]["config"][aggregated_field] data_dict[aggr_field_val] = {"min": min(times), "avg": sum(times) / len(times), "max": max(times)} aggr_field_vals = sorted(data_dict.keys()) mins = [data_dict[x]["min"] for x in aggr_field_vals] avgs = [data_dict[x]["avg"] for x in aggr_field_vals] maxes = [data_dict[x]["max"] for x in aggr_field_vals] axes = plt.subplot(111) plt.plot(aggr_field_vals, maxes, "r-", label="max", linewidth=2) plt.plot(aggr_field_vals, avgs, "b-", label="avg", linewidth=2) plt.plot(aggr_field_vals, mins, "g-", label="min", linewidth=2) title = "Benchmark results: %s" % benchmark_name plt.title(title) fig = plt.gcf() fig.canvas.set_window_title(title) plt.xlabel(aggregated_field) axes.set_xlim(0, max(aggr_field_vals) + 1) x_axis = axes.get_xaxis() x_axis.set_major_locator(ticker.MaxNLocator(integer=True)) plt.ylabel("Time (sec)") axes.set_ylim(min(mins) - 2, max(maxes) + 2) plt.legend(loc="upper right") plt.show()
def detailed(self, task_id): """Get detailed information about task :param task_id: Task uuid Prints detailed infomration of task. """ if task_id == "last": task = db.task_get_detailed_last() task_id = task.uuid else: task = db.task_get_detailed(task_id) print() print("=" * 80) print(_("Task %(task_id)s is %(status)s.") % {'task_id': task_id, 'status': task['status']}) for result in task["results"]: key = result["key"] print("-" * 80) print() print("test scenario %s" % key["name"]) print("args position %s" % key["pos"]) print("args values:") pprint.pprint(key["kw"]) raw = result["data"]["raw"] times = map(lambda x: x['time'], filter(lambda r: not r['error'], raw)) table = prettytable.PrettyTable(["max", "avg", "min", "ratio"]) if times: table.add_row([max(times), sum(times) / len(times), min(times), float(len(times)) / len(raw)]) else: table.add_row(['n/a', 'n/a', 'n/a', 0]) print(table) #NOTE(hughsaunders): ssrs=scenario specific results ssrs = [] for result in raw: try: ssrs.append(result['scenario_output']['data']) except (KeyError, TypeError): # No SSRs in this result pass if ssrs: sys.stdout.flush() keys = set() for ssr in ssrs: keys.update(ssr.keys()) ssr_table = prettytable.PrettyTable( ["Key", "max", "avg", "min"]) for key in keys: values = [float(ssr[key]) for ssr in ssrs if key in ssr] if values: row = [str(key), max(values), sum(values) / len(values), min(values)] else: row = [str(key)] + ['n/a'] * 3 ssr_table.add_row(row) print("\nScenario Specific Results\n") print(ssr_table) for result in raw: if result['scenario_output']['errors']: print(result['scenario_output']['errors'])
def detailed(self, task_id): """Get detailed information about task :param task_id: Task uuid Prints detailed information of task. """ def _print_atomic_actions_time(raw): atime_merged = [] for r in raw: if 'atomic_actions_time' in r: for a in r['atomic_actions_time']: atime_merged.append(a) times_by_action = collections.defaultdict(list) for at in atime_merged: times_by_action[at['action']].append(at['duration']) if times_by_action: atomic_action_table = prettytable.PrettyTable( ['action', 'count', 'max (sec)', 'avg (sec)', 'min (sec)', '90 percentile', '95 percentile']) for k, v in times_by_action.iteritems(): atomic_action_table.add_row([k, len(v), max(v), sum(v) / len(v), min(v), percentile(v, 0.90), percentile(v, 0.95)]) print(atomic_action_table) print() if task_id == "last": task = db.task_get_detailed_last() task_id = task.uuid else: task = db.task_get_detailed(task_id) if task is None: print("The task %s can not be found" % task_id) return print() print("=" * 80) print(_("Task %(task_id)s is %(status)s. Failed: %(failed)s") % {'task_id': task_id, 'status': task['status'], 'failed': task['failed'] }) for result in task["results"]: key = result["key"] print("-" * 80) print() print("test scenario %s" % key["name"]) print("args position %s" % key["pos"]) print("args values:") pprint.pprint(key["kw"]) if not result["data"]["validation"]["is_valid"]: print("-" * 80) print(result["data"]["validation"]["exc_msg"]) continue _print_atomic_actions_time(result["data"]["raw"]) raw = result["data"]["raw"] times = map(lambda x: x['time'], filter(lambda r: not r['error'], raw)) table = prettytable.PrettyTable(["max (sec)", "avg (sec)", "min (sec)", "90 pecentile", "95 percentile", "success/total", "total times"]) if times: table.add_row([max(times), sum(times) / len(times), min(times), percentile(times, 0.90), percentile(times, 0.95), float(len(times)) / len(raw), len(raw)]) else: table.add_row(['n/a', 'n/a', 'n/a', 'n/a', 'n/a', 0, len(raw)]) print(table) #NOTE(hughsaunders): ssrs=scenario specific results ssrs = [] for result in raw: try: ssrs.append(result['scenario_output']['data']) except (KeyError, TypeError): # No SSRs in this result pass if ssrs: sys.stdout.flush() keys = set() for ssr in ssrs: keys.update(ssr.keys()) ssr_table = prettytable.PrettyTable(["Key", "max", "avg", "min", "90 pecentile", "95 pecentile"]) for key in keys: values = [float(ssr[key]) for ssr in ssrs if key in ssr] if values: row = [str(key), max(values), sum(values) / len(values), min(values), percentile(values, 0.90), percentile(values, 0.95)] else: row = [str(key)] + ['n/a'] * 5 ssr_table.add_row(row) print("\nScenario Specific Results\n") print(ssr_table) for result in raw: if result['scenario_output']['errors']: print(result['scenario_output']['errors'])