def tabulate_metrics(metrics: List[PerfMetric]) -> None:
abbrevLen = get_abbrev_hash_length()
hasBaseline = any([x.baseline is not None for x in metrics])
baselineCommitSet = set(
[x.baseline.commit for x in metrics if x.baseline is not None])
hideBaselineCommit = not hasBaseline or len(baselineCommitSet) == 1
hideBaselineEnv = not hasBaseline or all([
x.stat.test_env == x.baseline.perfStat.test_env
for x in metrics if x.baseline is not None
])
def row(cells: Tuple[str, str, str, str, str, str, str, str]) -> List[str]:
return [
x for (idx, x) in enumerate(list(cells))
if (idx != 2 or not hideBaselineCommit) and (
idx != 3 or not hideBaselineEnv)
]
headerRows = [
row(("", "", "Baseline", "Baseline", "Baseline", "", "", "")),
row(("Test", "Metric", "commit", "environment", "value", "New value",
"Change", ""))
]
def strDiff(x: PerfMetric) -> str:
if x.baseline is None:
return ""
val0 = x.baseline.perfStat.value
val1 = x.stat.value
return "{:+2.1f}%".format(100 * (val1 - val0) / val0)
dataRows = [
row(("{}({})".format(x.stat.test,
x.stat.way), shorten_metric_name(x.stat.metric),
"{}".format(x.baseline.commit[:abbrevLen] if is_commit_hash(
x.baseline.commit) else x.baseline.commit) if x.baseline
is not None else "", "{}".format(x.baseline.perfStat.test_env)
if x.baseline is not None else "",
"{:13.1f}".format(x.baseline.perfStat.value) if
x.baseline is not None else "", "{:13.1f}".format(x.stat.value),
strDiff(x), "{}".format(x.change.hint())))
for x in sorted(metrics,
key=lambda m: (m.stat.test, m.stat.way, m.stat.metric))
]
print_table(headerRows, dataRows, 1)
print("")
if hasBaseline:
if hideBaselineEnv:
print(
"* All baselines were measured in the same environment as this test run"
)
if hideBaselineCommit:
commit = next(iter(baselineCommitSet))
print("* All baseline commits are {}".format(
commit[:abbrevLen] if is_commit_hash(commit) else commit))
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--add-note",
nargs=3,
help="Development only. --add-note N commit seed \
Adds N fake metrics to the given commit using the random seed."
)
parser.add_argument("--ci", action='store_true',
help="Use ci results. You must fetch these with:\n " \
+ "$ git fetch https://gitlab.haskell.org/ghc/ghc-performance-notes.git refs/notes/perf:refs/notes/ci/perf")
group = parser.add_argument_group(
title='Filtering',
description="Select which subset of performance metrics to dump")
group.add_argument(
"--test-env",
help=
"The given test environment to be compared. Use 'local' for locally run results. If using --ci, see .gitlab-ci file for TEST_ENV settings."
)
group.add_argument(
"--test-name",
help="Filters for tests matching the given regular expression.")
group.add_argument("--metric",
help="Test metric (one of " + str(testing_metrics()) +
").")
group.add_argument("--way",
help="Test way (one of " + str(testing_metrics()) +
").")
group = parser.add_argument_group(
title='Plotting', description="Plot historical performance metrics")
group.add_argument(
"--chart",
nargs='?',
default=None,
action='store',
const='./PerformanceChart.html',
help=
'Create a chart of the results an save it to the given file. Default to "./PerformanceChart.html".'
)
group.add_argument("--zero-y",
action='store_true',
help='When charting, include 0 in y axis')
parser.add_argument(
"commits",
nargs='+',
help=
"Either a list of commits or a single commit range (e.g. HEAD~10..HEAD)."
)
args = parser.parse_args()
env = 'local'
name = re.compile('.*')
CommitAndStat = NamedTuple('CommitAndStat', [('commit', GitHash),
('stat', PerfStat)])
metrics = [] # type: List[CommitAndStat]
singleton_commit = len(args.commits) == 1
#
# Main logic of the program when called from the command-line.
#
ref = NoteNamespace('perf')
if args.ci:
ref = NoteNamespace('ci/perf')
commits = args.commits
if args.commits:
# Commit range
if len(commits) == 1 and ".." in commits[0]:
commits = list(reversed(commit_log(commits[0])))
for c in commits:
metrics += [
CommitAndStat(c, stat) for stat in get_perf_stats(c, ref)
]
if args.metric:
metrics = [test for test in metrics if test.stat.metric == args.metric]
if args.way:
metrics = [test for test in metrics if test.stat.way == args.way]
if args.test_env:
if '"' in args.test_env:
raise Exception('test_env should not contain quotation marks')
metrics = [
test for test in metrics if test.stat.test_env == args.test_env
]
if args.test_name:
nameRe = re.compile(args.test_name)
metrics = [test for test in metrics if nameRe.search(test.stat.test)]
if args.add_note:
def note_gen(n, commit, delta=''):
note = []
# Generates simple fake data. Likely not comprehensive enough to catch all edge cases.
if not delta:
note.extend([
PerfStat('local', 'T' + str(i * 100), 'some_way',
'some_field', str(i * 1000))
for i in range(1,
int(int(n) / 2) + 1)
])
note.extend([
PerfStat('non-local', 'W' + str(i * 100), 'other_way',
'other_field', str(i * 100))
for i in range(int(int(n) / 2) + 1,
int(n) + 1)
])
if delta:
hu = abs(hash(delta))
hv = abs(hash(hu))
u = int(hu % 100)
v = int(hv % 10)
note.extend([
PerfStat('local', 'T' + str(i * 100), 'some_way',
'some_field', str(i * u))
for i in range(1,
int(int(n) / 2) + 1)
])
note.extend([
PerfStat('non-local', 'W' + str(i * 100), 'other_way',
'other_field', str(i * v))
for i in range(int(int(n) / 2) + 1,
int(n) + 1)
])
append_perf_stat(note, commit)
note_gen(args.add_note[0], args.add_note[1], args.add_note[2])
#
# Chart
#
def metricAt(commit, testName, testMetric):
values2 = [float(t.stat.value) for t in metrics if t.commit == commit \
and t.stat.test == testName \
and t.stat.metric == testMetric]
if values2 == []:
return None
else:
return (sum(values2) / len(values2))
testSeries = list(
set([(test.stat.test_env, test.stat.test, test.stat.metric,
test.stat.way) for test in metrics]))
#
# Use Chart.js to visualize the data.
#
if args.chart:
commitMsgs = dict([(h, get_commit_message(h)) for h in commits])
chartData = {
'type': 'line',
'data': {
'labels': [commitMsgs[h].split("\n")[0] + " (" + \
(h[:8] if is_commit_hash(h) else h) + \
")" for h in commits],
'datasets': [{
'label': name + "(" + way + ") " + metric + " - " + env,
'data': [get_commit_metric_value_str_or_none(ref, commit, env, name, metric, way) \
for commit in commits],
'fill': 'false',
'spanGaps': 'true',
'lineTension': 0,
'backgroundColor': hash_rgb_str((env, name, metric, way)),
'borderColor': hash_rgb_str((env, name, metric, way))
} for (env, name, metric, way) in testSeries]
},
'options': {
'scales': {
'yAxes': [{
'ticks': { 'beginAtZero': True }
}]
}
}
}
# Try use local Chart.js file else use online version.
tooltipjsFilePath = sys.path[0] + "/js/tooltip.js"
chartjsFilePath = sys.path[0] + "/js/Chart-2.8.0.min.js"
tooltipjsTag = None
try:
tooltipjsFile = open(tooltipjsFilePath, "r")
tooltipjsTag = '<script>' + tooltipjsFile.read() + '</script>'
tooltipjsFile.close()
except:
print("Failed to load custom tooltip: " + chartjsFilePath + ".")
tooltipjsTag = None
try:
chartjsFile = open(chartjsFilePath, "r")
chartjsTag = '<script>' + chartjsFile.read() + '</script>'
chartjsFile.close()
except:
print("Failed to load " + chartjsFilePath +
", reverting to online Chart.js.")
chartjsTag = '<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>'
file = open(args.chart, "w+t")
print(\
"<html>" + \
'<head>\n' + \
(tooltipjsTag if tooltipjsTag is not None else '') + \
chartjsTag + \
'</head>' + \
'<body style="padding: 20px"><canvas id="myChart"></canvas><script>' + \
"var ctx = document.getElementById('myChart').getContext('2d');" + \
"var commitMsgs = " + json.dumps(commitMsgs, indent=2) + ";" + \
"var chartData = " + json.dumps(chartData, indent=2) + ";" + \
(("var chart = new Chart(ctx, setCustomTooltip(chartData, commitMsgs));") \
if tooltipjsTag is not None else \
("var chart = new Chart(ctx, chartData);")) + \
'</script></body>' + \
"</html>"\
, file=file)
file.close()
exit(0)
#
# Print the data in tablular format
#
# T1234 T1234
# max_bytes max_bytes
# normal normal
# commit x86_64-darwin i386-linux-deb9
# --------------------------------------------
# HEAD 9123 9123
# HEAD~1 10023 10023
# HEAD~2 21234 21234
# HEAD~3 20000 20000
def strMetric(x):
return '{:.2f}'.format(x.value) if x != None else ""
# Data is in colum major format, so transpose and pass to print_table.
T = TypeVar('T')
def transpose(xss: List[List[T]]) -> List[List[T]]:
return list(map(list, zip(*xss)))
headerCols = [ ["","","","Commit"] ] \
+ [ [name, metric, way, env] for (env, name, metric, way) in testSeries ]
dataCols = [ commits ] \
+ [ [strMetric(get_commit_metric(ref, commit, env, name, metric, way)) \
for commit in commits ] \
for (env, name, metric, way) in testSeries ]
print_table(transpose(headerCols), transpose(dataCols))
