def get_data(name='',
type='',
x_field='revision',
y_field='runtime',
last_num=-1):
"""Get the test runtime/iteration as a function of an X variable.
Parameters
----------
name :: full name of the test
type :: type of test to filter by
x_field :: name of the field for the X axis.
e.g. 'revision' (default)
or 'date' : exact date/time of launch
or 'index' : using the date, but returning an index of build #
instead of the date (better scaling)
last_num :: only get the last this-many entries from the table, sorted by revision.
if < 0, then get everything
Returns
-------
x :: list of X values, sorted increasing
y :: list of runtime/iteration for each x
"""
results = get_results(name,
type,
where_clause='',
orderby_clause=get_orderby_clause(last_num))
# Data dict. Key = X variable; Value = (iterations total, runtime total)
data = {}
for res in results:
# Get the x field value
if x_field == 'index':
x = res['date']
else:
x = res[x_field]
if data.has_key(x):
old = data[x]
iters = old[0] + 1 # Iterations
runtime = old[1] + res[y_field]
else:
iters = 1
runtime = res[y_field]
# Save the # of iterations and runtime
data[x] = (iters, runtime)
# Now make a sorted list of (x, runtime/iteration)
sorted = [(x, y[1] / y[0]) for (x, y) in data.items()]
sorted.sort()
x = [a for (a, b) in sorted]
# For index, convert into an integer index
if x_field == 'index':
x = range(len(x))
y = [b for (a, b) in sorted]
return (x, y)
def get_runtime_data(name='', type='', x_field='revision', last_num=-1):
"""Get the test runtime/iteration as a function of an X variable.
Parameters
----------
name :: full name of the test
type :: type of test to filter by
x_field :: name of the field for the X axis.
e.g. 'revision' (default)
or 'date' : exact date/time of launch
or 'index' : using the date, but returning an index of build #
instead of the date (better scaling)
last_num :: only get the last this-many entries from the table, sorted by revision.
if < 0, then get everything
Returns
-------
x :: list of X values, sorted increasing
y :: list of runtime/iteration for each x
"""
results = get_results(name, type, where_clause='', orderby_clause=get_orderby_clause(last_num))
# Data dict. Key = X variable; Value = (iterations total, runtime total)
data = {}
for res in results:
# Get the x field value
if x_field == 'index':
x = res['date']
else:
x = res[x_field]
if data.has_key(x):
old = data[x]
iters = old[0] + 1 # Iterations
runtime = old[1] + res["runtime"]
else:
iters = 1
runtime = res["runtime"]
# Save the # of iterations and runtime
data[x] = (iters, runtime)
# Now make a sorted list of (x, runtime/iteration)
sorted = [(x, y[1]/y[0]) for (x,y) in data.items()]
sorted.sort()
x = [a for (a,b) in sorted]
# For index, convert into an integer index
if x_field == 'index':
x = range( len(x) )
y = [b for (a,b) in sorted]
return (x,y)
def plot_success_count(type='system', last_num=-1, x_field='revision'):
""" Plot the count of successful/failed tests vs revision number
Parameters
----------
type :: 'system', or 'performance'
"""
results = get_results('',
type,
where_clause='',
orderby_clause=get_orderby_clause(last_num))
revisions = get_unique_fields(results, x_field)
# Go through each revision
success = []
fail = []
for revision in revisions:
these = get_results_matching(results, x_field, revision)
succeeded = 0
failed = 0
for res in these:
if res["success"]:
succeeded += 1
else:
failed += 1
# Keep the list of them
success.append(succeeded)
fail.append(failed)
figure()
revisions = np.array(revisions)
fail = np.array(fail)
success = np.array(success)
index = np.arange(len(revisions))
# p1 = bar(index, fail, color='r')
# p2 = bar(index, success, color='g', bottom=fail)
# legend( (p1[0], p2[0]), ('Failure', 'Success') )
p1 = fill_between(index, fail, 0, color='r')
p2 = fill_between(index, success + fail, fail, color='g')
#legend( (p1, p2), ('Failure', 'Success') )
smart_ticks(index, revisions)
ylabel('Success/Fail')
xlabel(x_field)
revsare = "all revs"
if last_num > 0: revsare = "last %d revs" % last_num
title("Success/Fail History of %s tests (%s)" % (type, revsare))
def plot_success_count(type='system', last_num=-1, x_field='revision'):
""" Plot the count of successful/failed tests vs revision number
Parameters
----------
type :: 'system', or 'performance'
"""
results = get_results('', type, where_clause='', orderby_clause=get_orderby_clause(last_num))
revisions = get_unique_fields(results, x_field)
# Go through each revision
success = []
fail = []
for revision in revisions:
these = get_results_matching(results, x_field, revision)
succeeded = 0
failed = 0
for res in these:
if res["success"]:
succeeded += 1
else:
failed += 1
# Keep the list of them
success.append(succeeded)
fail.append(failed)
figure()
revisions = np.array(revisions)
fail = np.array(fail)
success = np.array(success)
index = np.arange(len(revisions))
# p1 = bar(index, fail, color='r')
# p2 = bar(index, success, color='g', bottom=fail)
# legend( (p1[0], p2[0]), ('Failure', 'Success') )
p1 = fill_between(index, fail, 0, color='r')
p2 = fill_between(index, success+fail, fail, color='g')
#legend( (p1, p2), ('Failure', 'Success') )
smart_ticks( index, revisions)
ylabel('Success/Fail')
xlabel(x_field)
revsare = "all revs"
if last_num > 0: revsare = "last %d revs" % last_num
title("Success/Fail History of %s tests (%s)" % (type, revsare))
def make_detailed_html_file(basedir, name, fig1, fig2, last_num):
""" Create a detailed HTML report for the named test """
html = DEFAULT_PLOTLY_HEADER
html += """<h1>Detailed report for %s</h1><br>""" % (name)
html += fig1 + "\n"
html += fig2 + "\n"
html += """<h3>Test Results</h3>"""
fields = [
'revision', 'date', 'commitid', 'compare', 'status', 'runtime',
'cpu_fraction', 'variables'
]
table_row_header = "<tr>"
for field in fields:
if field == "runtime": field = "Runtime/Iter."
field = field[0].upper() + field[1:]
table_row_header += "<th>%s</th>" % field
table_row_header += "</tr>"
html += """<table border="1">""" + table_row_header
table_html = ''
results = get_results(name,
type='',
where_clause='',
orderby_clause="ORDER BY revision, variables, date")
data = [(res["revision"], res["variables"], res["date"], res)
for res in results]
count = 0
last_rev = 0
commitid = ''
last_commitid = ''
row_class = ''
table_rows = []
for (rev, variable, date, res) in data:
table_row_html = ''
if (rev != last_rev):
# Changed SVN revision. Swap row color
if row_class == '':
row_class = "class=alternaterow"
else:
row_class = ''
last_rev = rev
if commitid != last_commitid:
last_commitid = commitid
if res["success"]:
table_row_html += "<tr %s>\n" % row_class
else:
table_row_html += "<tr class=failedrow>\n"
for field in fields:
val = ''
if field == 'compare':
# Comparison to previous commit, if anything can be done
if (last_commitid != ""):
val = """<a href="%s/compare/%s...%s">diff</a>""" % (
MANTID_ADDRESS, last_commitid, commitid)
else:
# Normal fields
val = res[field]
# Trim the fractional seconds
if field == "date":
val = str(val)[0:19]
# Add a trac link
if field == "commitid":
commitid = val
partial_commitid = val
if (len(partial_commitid) > 7):
partial_commitid = partial_commitid[0:7]
val = """<a href="%s/commit/%s">%s</a>""" % (
MANTID_ADDRESS, commitid, partial_commitid)
if field == "runtime":
val = "%.3f" % (res["runtime"])
table_row_html += "<td>%s</td>" % val
table_row_html += "\n</tr>\n"
table_rows.append(table_row_html)
# Now print out all the rows in reverse order
table_rows.reverse()
for row in table_rows:
html += row
# And one more at the end for good measure
html += table_row_header
html += "</table>"
if results:
html += """<h3>Environment</h3>
%s""" % make_environment_html(results[0])
html += DEFAULT_HTML_FOOTER
f = open(os.path.join(basedir, "%s.htm" % name), "w")
html = html.replace("\n", os.linesep) # Fix line endings for windows
f.write(html)
f.close()
def plot_runtime(annotate, saveImage, path, **kwargs):
name = kwargs['name']
(xData, yData) = get_runtime_data(**kwargs)
trace1 = go.Scatter(x=xData,
y=yData,
mode='lines+markers',
marker=dict(size='5', color="blue"))
annotations = []
last_num = kwargs.get('last_num', None)
if annotate and not saveImage:
# retrieve commitids for annotation on the plotly graph
results = get_results(
name, orderby_clause='ORDER BY revision, variables, date')
commitids = [
"""<a href="%s/commit/%s"> </a>""" %
(MANTID_ADDRESS, res["commitid"]) for res in results
]
if last_num is not None:
commitids = commitids[-last_num:]
for x, y, text in zip(xData, yData, commitids):
annotations.append(
dict(x=x,
y=y,
text=text,
showarrow=False,
font=dict(family='sans serif', size=10),
xanchor='center',
yanchor='center'))
if last_num is not None:
title = "Runtime History of %s (last %d revs)" % (name, last_num)
else:
title = "Runtime History of %s (all revs)" % name
yAxisTitle = 'Runtime/iteration (sec)'
xAxisTitle = kwargs['x_field']
layout = go.Layout(showlegend=False,
annotations=annotations,
title=title,
xaxis=dict(title=xAxisTitle),
yaxis=dict(title=yAxisTitle, range=[0,
np.amax(yData)]))
data = [trace1]
fig = go.Figure(data=data, layout=layout)
if saveImage:
im_filename = name + ".png"
plt.ioff()
plt.figure()
plt.title(title)
plt.xlabel(xAxisTitle)
plt.ylabel(yAxisTitle)
plt.plot(xData, yData, "-b.")
plt.ylim(ymin=0)
plt.savefig(os.path.join(path, im_filename))
plt.close()
return """<img src="%s"/>""" % im_filename
else:
return offline.plot(fig,
output_type='div',
show_link=False,
auto_open=False,
include_plotlyjs=False)
def make_detailed_html_file(basedir, name, fig1, fig2, last_num):
""" Create a detailed HTML report for the named test """
html = default_html_header
html += """<h1>Detailed report for %s</h1><br>""" % (name)
html += """<img src="%s" alt="runtime vs revision number (latest %d entries)" />\n""" % (fig1, last_num)
html += """<img src="%s" alt="runtime vs revision number" />\n""" % (fig2)
html += """<h3>Test Results</h3>"""
fields = ['revision', 'date', 'commitid', 'compare', 'status', 'runtime', 'cpu_fraction', 'variables']
table_row_header = "<tr>"
for field in fields:
if field == "runtime": field = "Runtime/Iter."
field = field[0].upper() + field[1:]
table_row_header += "<th>%s</th>" % field
table_row_header += "</tr>"
html += """<table border="1">""" + table_row_header
table_html = ''
results = get_results(name, type='', where_clause='')
sorted = [(res["revision"], res["variables"], res["date"], res) for res in results]
sorted.sort(reverse=False)
count = 0
last_rev = 0
commitid = ''
last_commitid = ''
row_class = ''
table_rows = []
for (rev, variable, date, res) in sorted:
table_row_html = ''
if (rev != last_rev):
# Changed SVN revision. Swap row color
if row_class == '':
row_class = "class=alternaterow"
else:
row_class = ''
last_rev = rev
if commitid != last_commitid:
last_commitid = commitid
if res["success"]:
table_row_html += "<tr %s>\n" % row_class
else:
table_row_html += "<tr class=failedrow>\n"
for field in fields:
val = ''
if field == 'compare':
# Comparison to previous commit, if anything can be done
if (last_commitid != ""):
val = """<a href="https://github.com/mantidproject/mantid/compare/%s...%s">diff</a>""" % (last_commitid, commitid)
else:
# Normal fields
val = res[field]
# Trim the fractional seconds
if field=="date":
val = str(val)[0:19]
# Add a trac link
if field=="commitid":
commitid = val
partial_commitid = val
if (len(partial_commitid) > 7): partial_commitid = partial_commitid[0:7];
val = """<a href="https://github.com/mantidproject/mantid/commit/%s">%s</a>""" % (commitid, partial_commitid)
if field=="runtime":
val = "%.3f" % (res["runtime"])
table_row_html += "<td>%s</td>" % val
table_row_html += "\n</tr>\n"
table_rows.append(table_row_html)
# Now print out all the rows in reverse order
table_rows.reverse()
for row in table_rows:
html += row
# # Add the row header every 30 entries
# count += 1
# if count % 30 == 0: html += table_row_header
# And one more at the end for good measure
html += table_row_header
html += "</table>"
if len(results)> 0:
html += """<h3>Environment</h3>
%s""" % make_environment_html(results[0])
html += default_html_footer
# last_date = sorted[-1][1]["date"]
# results = get_results(name, type='', get_log=False, where_clause=" date = '%s'" % last_date)
# if len(results)>0:
# html +=
f = open(os.path.join(basedir, "%s.htm" % name), "w")
html = html.replace("\n", os.linesep) # Fix line endings for windows
f.write(html)
f.close()
def make_detailed_html_file(basedir, name, fig1, fig2, last_num):
""" Create a detailed HTML report for the named test """
html = default_html_header
html += """<h1>Detailed report for %s</h1><br>""" % (name)
html += """<img src="%s" alt="runtime vs revision number (latest %d entries)" />\n""" % (fig1, last_num)
html += """<img src="%s" alt="runtime vs revision number" />\n""" % (fig2)
html += """<h3>Test Results</h3>"""
fields = ['date', 'revision', 'status', 'iterations', 'runtime', 'cpu_fraction', 'speed_up', 'variables']
table_row_header = "<tr>"
for field in fields:
if field == "runtime": field = "Runtime/Iter."
field = field[0].upper() + field[1:]
table_row_header += "<th>%s</th>" % field
table_row_header += "</tr>"
html += """<table border="1">""" + table_row_header
results = get_results(name, type='', get_log=False, where_clause='')
sorted = [(res["revision"], res["variables"], res["date"], res) for res in results]
sorted.sort(reverse=True)
count = 0
last_rev = 0
row_class = ''
for (rev, variable, date, res) in sorted:
if (rev != last_rev):
# Changed SVN revision. Swap row color
if row_class == '':
row_class = "class=alternaterow"
else:
row_class = ''
last_rev = rev
if res["success"]:
html += "<tr %s>\n" % row_class
else:
html += "<tr class=failedrow>\n"
for field in fields:
val = res[field]
# Trim the fractional seconds
if field=="date":
val = str(val)[0:19]
# Add a trac link
if field=="revision":
try:
num = int(val)
except:
num = 0
val = """<a href="http://trac.mantidproject.org/mantid/changeset/%d">%d</a>""" % (num, num)
if field=="runtime":
val = "%.3f" % (res["runtime"]/res["iterations"])
html += "<td>%s</td>" % val
html += "\n</tr>\n"
# # Add the row header every 30 entries
# count += 1
# if count % 30 == 0: html += table_row_header
# And one more at the end for good measure
html += table_row_header
html += "</table>"
if len(results)> 0:
html += """<h3>Environment</h3>
%s""" % make_environment_html(results[0])
html += default_html_footer
# last_date = sorted[-1][1]["date"]
# results = get_results(name, type='', get_log=False, where_clause=" date = '%s'" % last_date)
# if len(results)>0:
# html +=
f = open(os.path.join(basedir, "%s.htm" % name), "w")
f.write(html)
f.close()
def make_detailed_html_file(basedir, name, fig1, fig2, last_num):
""" Create a detailed HTML report for the named test """
html = DEFAULT_PLOTLY_HEADER
html += """<h1>Detailed report for %s</h1><br>""" % (name)
html += fig1 + "\n"
html += fig2 + "\n"
html += """<h3>Test Results</h3>"""
fields = ['revision', 'date', 'commitid', 'compare', 'status', 'runtime', 'cpu_fraction', 'variables']
table_row_header = "<tr>"
for field in fields:
if field == "runtime": field = "Runtime/Iter."
field = field[0].upper() + field[1:]
table_row_header += "<th>%s</th>" % field
table_row_header += "</tr>"
html += """<table border="1">""" + table_row_header
table_html = ''
results = get_results(name, type='', where_clause='', orderby_clause="ORDER BY revision, variables, date")
data = [(res["revision"], res["variables"], res["date"], res) for res in results]
count = 0
last_rev = 0
commitid = ''
last_commitid = ''
row_class = ''
table_rows = []
for (rev, variable, date, res) in data:
table_row_html = ''
if (rev != last_rev):
# Changed SVN revision. Swap row color
if row_class == '':
row_class = "class=alternaterow"
else:
row_class = ''
last_rev = rev
if commitid != last_commitid:
last_commitid = commitid
if res["success"]:
table_row_html += "<tr %s>\n" % row_class
else:
table_row_html += "<tr class=failedrow>\n"
for field in fields:
val = ''
if field == 'compare':
# Comparison to previous commit, if anything can be done
if (last_commitid != ""):
val = """<a href="%s/compare/%s...%s">diff</a>""" % (
MANTID_ADDRESS, last_commitid, commitid)
else:
# Normal fields
val = res[field]
# Trim the fractional seconds
if field == "date":
val = str(val)[0:19]
# Add a trac link
if field == "commitid":
commitid = val
partial_commitid = val
if (len(partial_commitid) > 7): partial_commitid = partial_commitid[0:7];
val = """<a href="%s/commit/%s">%s</a>""" % (MANTID_ADDRESS, commitid, partial_commitid)
if field == "runtime":
val = "%.3f" % (res["runtime"])
table_row_html += "<td>%s</td>" % val
table_row_html += "\n</tr>\n"
table_rows.append(table_row_html)
# Now print out all the rows in reverse order
table_rows.reverse()
for row in table_rows:
html += row
# And one more at the end for good measure
html += table_row_header
html += "</table>"
if results:
html += """<h3>Environment</h3>
%s""" % make_environment_html(results[0])
html += DEFAULT_HTML_FOOTER
f = open(os.path.join(basedir, "%s.htm" % name), "w")
html = html.replace("\n", os.linesep) # Fix line endings for windows
f.write(html)
f.close()
def plot_runtime(annotate, saveImage, path, **kwargs):
name = kwargs['name']
(xData, yData) = get_runtime_data(**kwargs)
trace1 = go.Scatter(x=xData, y=yData,
mode='lines+markers',
marker=dict(
size='5',
color="blue"
)
)
annotations = []
last_num = kwargs.get('last_num', None)
if annotate and not saveImage:
# retrieve commitids for annotation on the plotly graph
results = get_results(name, orderby_clause='ORDER BY revision, variables, date')
commitids = ["""<a href="%s/commit/%s"> </a>""" % (MANTID_ADDRESS, res["commitid"]) for res in results]
if last_num is not None:
commitids = commitids[-last_num:]
for x, y, text in zip(xData, yData, commitids):
annotations.append(
dict(
x=x,
y=y,
text=text,
showarrow=False,
font=dict(family='sans serif', size=10),
xanchor='center',
yanchor='center'
)
)
if last_num is not None:
title = "Runtime History of %s (last %d revs)" % (name, last_num)
else:
title = "Runtime History of %s (all revs)" % name
yAxisTitle = 'Runtime/iteration (sec)'
xAxisTitle = kwargs['x_field']
layout = go.Layout(showlegend=False, annotations=annotations,
title=title,
xaxis=dict(title=xAxisTitle),
yaxis=dict(
title=yAxisTitle,
range=[0, np.amax(yData)]
)
)
data = [trace1]
fig = go.Figure(data=data, layout=layout)
if saveImage:
im_filename = name + ".png"
plt.ioff()
plt.figure()
plt.title(title)
plt.xlabel(xAxisTitle)
plt.ylabel(yAxisTitle)
plt.plot(xData, yData, "-b.")
plt.ylim(ymin=0)
plt.savefig(os.path.join(path, im_filename))
plt.close()
return """<img src="%s"/>""" % im_filename
else:
return offline.plot(fig, output_type='div', show_link=False, auto_open=False, include_plotlyjs=False)
