def get_data(name='', type='', x_field='revision', y_field='runtime', last_num=-1): """Get the test runtime/iteration as a function of an X variable. Parameters ---------- name :: full name of the test type :: type of test to filter by x_field :: name of the field for the X axis. e.g. 'revision' (default) or 'date' : exact date/time of launch or 'index' : using the date, but returning an index of build # instead of the date (better scaling) last_num :: only get the last this-many entries from the table, sorted by revision. if < 0, then get everything Returns ------- x :: list of X values, sorted increasing y :: list of runtime/iteration for each x """ results = get_results(name, type, where_clause='', orderby_clause=get_orderby_clause(last_num)) # Data dict. Key = X variable; Value = (iterations total, runtime total) data = {} for res in results: # Get the x field value if x_field == 'index': x = res['date'] else: x = res[x_field] if data.has_key(x): old = data[x] iters = old[0] + 1 # Iterations runtime = old[1] + res[y_field] else: iters = 1 runtime = res[y_field] # Save the # of iterations and runtime data[x] = (iters, runtime) # Now make a sorted list of (x, runtime/iteration) sorted = [(x, y[1] / y[0]) for (x, y) in data.items()] sorted.sort() x = [a for (a, b) in sorted] # For index, convert into an integer index if x_field == 'index': x = range(len(x)) y = [b for (a, b) in sorted] return (x, y)
def get_runtime_data(name='', type='', x_field='revision', last_num=-1): """Get the test runtime/iteration as a function of an X variable. Parameters ---------- name :: full name of the test type :: type of test to filter by x_field :: name of the field for the X axis. e.g. 'revision' (default) or 'date' : exact date/time of launch or 'index' : using the date, but returning an index of build # instead of the date (better scaling) last_num :: only get the last this-many entries from the table, sorted by revision. if < 0, then get everything Returns ------- x :: list of X values, sorted increasing y :: list of runtime/iteration for each x """ results = get_results(name, type, where_clause='', orderby_clause=get_orderby_clause(last_num)) # Data dict. Key = X variable; Value = (iterations total, runtime total) data = {} for res in results: # Get the x field value if x_field == 'index': x = res['date'] else: x = res[x_field] if data.has_key(x): old = data[x] iters = old[0] + 1 # Iterations runtime = old[1] + res["runtime"] else: iters = 1 runtime = res["runtime"] # Save the # of iterations and runtime data[x] = (iters, runtime) # Now make a sorted list of (x, runtime/iteration) sorted = [(x, y[1]/y[0]) for (x,y) in data.items()] sorted.sort() x = [a for (a,b) in sorted] # For index, convert into an integer index if x_field == 'index': x = range( len(x) ) y = [b for (a,b) in sorted] return (x,y)
def plot_success_count(type='system', last_num=-1, x_field='revision'): """ Plot the count of successful/failed tests vs revision number Parameters ---------- type :: 'system', or 'performance' """ results = get_results('', type, where_clause='', orderby_clause=get_orderby_clause(last_num)) revisions = get_unique_fields(results, x_field) # Go through each revision success = [] fail = [] for revision in revisions: these = get_results_matching(results, x_field, revision) succeeded = 0 failed = 0 for res in these: if res["success"]: succeeded += 1 else: failed += 1 # Keep the list of them success.append(succeeded) fail.append(failed) figure() revisions = np.array(revisions) fail = np.array(fail) success = np.array(success) index = np.arange(len(revisions)) # p1 = bar(index, fail, color='r') # p2 = bar(index, success, color='g', bottom=fail) # legend( (p1[0], p2[0]), ('Failure', 'Success') ) p1 = fill_between(index, fail, 0, color='r') p2 = fill_between(index, success + fail, fail, color='g') #legend( (p1, p2), ('Failure', 'Success') ) smart_ticks(index, revisions) ylabel('Success/Fail') xlabel(x_field) revsare = "all revs" if last_num > 0: revsare = "last %d revs" % last_num title("Success/Fail History of %s tests (%s)" % (type, revsare))
def plot_success_count(type='system', last_num=-1, x_field='revision'): """ Plot the count of successful/failed tests vs revision number Parameters ---------- type :: 'system', or 'performance' """ results = get_results('', type, where_clause='', orderby_clause=get_orderby_clause(last_num)) revisions = get_unique_fields(results, x_field) # Go through each revision success = [] fail = [] for revision in revisions: these = get_results_matching(results, x_field, revision) succeeded = 0 failed = 0 for res in these: if res["success"]: succeeded += 1 else: failed += 1 # Keep the list of them success.append(succeeded) fail.append(failed) figure() revisions = np.array(revisions) fail = np.array(fail) success = np.array(success) index = np.arange(len(revisions)) # p1 = bar(index, fail, color='r') # p2 = bar(index, success, color='g', bottom=fail) # legend( (p1[0], p2[0]), ('Failure', 'Success') ) p1 = fill_between(index, fail, 0, color='r') p2 = fill_between(index, success+fail, fail, color='g') #legend( (p1, p2), ('Failure', 'Success') ) smart_ticks( index, revisions) ylabel('Success/Fail') xlabel(x_field) revsare = "all revs" if last_num > 0: revsare = "last %d revs" % last_num title("Success/Fail History of %s tests (%s)" % (type, revsare))
def make_detailed_html_file(basedir, name, fig1, fig2, last_num): """ Create a detailed HTML report for the named test """ html = DEFAULT_PLOTLY_HEADER html += """<h1>Detailed report for %s</h1><br>""" % (name) html += fig1 + "\n" html += fig2 + "\n" html += """<h3>Test Results</h3>""" fields = [ 'revision', 'date', 'commitid', 'compare', 'status', 'runtime', 'cpu_fraction', 'variables' ] table_row_header = "<tr>" for field in fields: if field == "runtime": field = "Runtime/Iter." field = field[0].upper() + field[1:] table_row_header += "<th>%s</th>" % field table_row_header += "</tr>" html += """<table border="1">""" + table_row_header table_html = '' results = get_results(name, type='', where_clause='', orderby_clause="ORDER BY revision, variables, date") data = [(res["revision"], res["variables"], res["date"], res) for res in results] count = 0 last_rev = 0 commitid = '' last_commitid = '' row_class = '' table_rows = [] for (rev, variable, date, res) in data: table_row_html = '' if (rev != last_rev): # Changed SVN revision. Swap row color if row_class == '': row_class = "class=alternaterow" else: row_class = '' last_rev = rev if commitid != last_commitid: last_commitid = commitid if res["success"]: table_row_html += "<tr %s>\n" % row_class else: table_row_html += "<tr class=failedrow>\n" for field in fields: val = '' if field == 'compare': # Comparison to previous commit, if anything can be done if (last_commitid != ""): val = """<a href="%s/compare/%s...%s">diff</a>""" % ( MANTID_ADDRESS, last_commitid, commitid) else: # Normal fields val = res[field] # Trim the fractional seconds if field == "date": val = str(val)[0:19] # Add a trac link if field == "commitid": commitid = val partial_commitid = val if (len(partial_commitid) > 7): partial_commitid = partial_commitid[0:7] val = """<a href="%s/commit/%s">%s</a>""" % ( MANTID_ADDRESS, commitid, partial_commitid) if field == "runtime": val = "%.3f" % (res["runtime"]) table_row_html += "<td>%s</td>" % val table_row_html += "\n</tr>\n" table_rows.append(table_row_html) # Now print out all the rows in reverse order table_rows.reverse() for row in table_rows: html += row # And one more at the end for good measure html += table_row_header html += "</table>" if results: html += """<h3>Environment</h3> %s""" % make_environment_html(results[0]) html += DEFAULT_HTML_FOOTER f = open(os.path.join(basedir, "%s.htm" % name), "w") html = html.replace("\n", os.linesep) # Fix line endings for windows f.write(html) f.close()
def plot_runtime(annotate, saveImage, path, **kwargs): name = kwargs['name'] (xData, yData) = get_runtime_data(**kwargs) trace1 = go.Scatter(x=xData, y=yData, mode='lines+markers', marker=dict(size='5', color="blue")) annotations = [] last_num = kwargs.get('last_num', None) if annotate and not saveImage: # retrieve commitids for annotation on the plotly graph results = get_results( name, orderby_clause='ORDER BY revision, variables, date') commitids = [ """<a href="%s/commit/%s"> </a>""" % (MANTID_ADDRESS, res["commitid"]) for res in results ] if last_num is not None: commitids = commitids[-last_num:] for x, y, text in zip(xData, yData, commitids): annotations.append( dict(x=x, y=y, text=text, showarrow=False, font=dict(family='sans serif', size=10), xanchor='center', yanchor='center')) if last_num is not None: title = "Runtime History of %s (last %d revs)" % (name, last_num) else: title = "Runtime History of %s (all revs)" % name yAxisTitle = 'Runtime/iteration (sec)' xAxisTitle = kwargs['x_field'] layout = go.Layout(showlegend=False, annotations=annotations, title=title, xaxis=dict(title=xAxisTitle), yaxis=dict(title=yAxisTitle, range=[0, np.amax(yData)])) data = [trace1] fig = go.Figure(data=data, layout=layout) if saveImage: im_filename = name + ".png" plt.ioff() plt.figure() plt.title(title) plt.xlabel(xAxisTitle) plt.ylabel(yAxisTitle) plt.plot(xData, yData, "-b.") plt.ylim(ymin=0) plt.savefig(os.path.join(path, im_filename)) plt.close() return """<img src="%s"/>""" % im_filename else: return offline.plot(fig, output_type='div', show_link=False, auto_open=False, include_plotlyjs=False)
def make_detailed_html_file(basedir, name, fig1, fig2, last_num): """ Create a detailed HTML report for the named test """ html = default_html_header html += """<h1>Detailed report for %s</h1><br>""" % (name) html += """<img src="%s" alt="runtime vs revision number (latest %d entries)" />\n""" % (fig1, last_num) html += """<img src="%s" alt="runtime vs revision number" />\n""" % (fig2) html += """<h3>Test Results</h3>""" fields = ['revision', 'date', 'commitid', 'compare', 'status', 'runtime', 'cpu_fraction', 'variables'] table_row_header = "<tr>" for field in fields: if field == "runtime": field = "Runtime/Iter." field = field[0].upper() + field[1:] table_row_header += "<th>%s</th>" % field table_row_header += "</tr>" html += """<table border="1">""" + table_row_header table_html = '' results = get_results(name, type='', where_clause='') sorted = [(res["revision"], res["variables"], res["date"], res) for res in results] sorted.sort(reverse=False) count = 0 last_rev = 0 commitid = '' last_commitid = '' row_class = '' table_rows = [] for (rev, variable, date, res) in sorted: table_row_html = '' if (rev != last_rev): # Changed SVN revision. Swap row color if row_class == '': row_class = "class=alternaterow" else: row_class = '' last_rev = rev if commitid != last_commitid: last_commitid = commitid if res["success"]: table_row_html += "<tr %s>\n" % row_class else: table_row_html += "<tr class=failedrow>\n" for field in fields: val = '' if field == 'compare': # Comparison to previous commit, if anything can be done if (last_commitid != ""): val = """<a href="https://github.com/mantidproject/mantid/compare/%s...%s">diff</a>""" % (last_commitid, commitid) else: # Normal fields val = res[field] # Trim the fractional seconds if field=="date": val = str(val)[0:19] # Add a trac link if field=="commitid": commitid = val partial_commitid = val if (len(partial_commitid) > 7): partial_commitid = partial_commitid[0:7]; val = """<a href="https://github.com/mantidproject/mantid/commit/%s">%s</a>""" % (commitid, partial_commitid) if field=="runtime": val = "%.3f" % (res["runtime"]) table_row_html += "<td>%s</td>" % val table_row_html += "\n</tr>\n" table_rows.append(table_row_html) # Now print out all the rows in reverse order table_rows.reverse() for row in table_rows: html += row # # Add the row header every 30 entries # count += 1 # if count % 30 == 0: html += table_row_header # And one more at the end for good measure html += table_row_header html += "</table>" if len(results)> 0: html += """<h3>Environment</h3> %s""" % make_environment_html(results[0]) html += default_html_footer # last_date = sorted[-1][1]["date"] # results = get_results(name, type='', get_log=False, where_clause=" date = '%s'" % last_date) # if len(results)>0: # html += f = open(os.path.join(basedir, "%s.htm" % name), "w") html = html.replace("\n", os.linesep) # Fix line endings for windows f.write(html) f.close()
def make_detailed_html_file(basedir, name, fig1, fig2, last_num): """ Create a detailed HTML report for the named test """ html = default_html_header html += """<h1>Detailed report for %s</h1><br>""" % (name) html += """<img src="%s" alt="runtime vs revision number (latest %d entries)" />\n""" % (fig1, last_num) html += """<img src="%s" alt="runtime vs revision number" />\n""" % (fig2) html += """<h3>Test Results</h3>""" fields = ['date', 'revision', 'status', 'iterations', 'runtime', 'cpu_fraction', 'speed_up', 'variables'] table_row_header = "<tr>" for field in fields: if field == "runtime": field = "Runtime/Iter." field = field[0].upper() + field[1:] table_row_header += "<th>%s</th>" % field table_row_header += "</tr>" html += """<table border="1">""" + table_row_header results = get_results(name, type='', get_log=False, where_clause='') sorted = [(res["revision"], res["variables"], res["date"], res) for res in results] sorted.sort(reverse=True) count = 0 last_rev = 0 row_class = '' for (rev, variable, date, res) in sorted: if (rev != last_rev): # Changed SVN revision. Swap row color if row_class == '': row_class = "class=alternaterow" else: row_class = '' last_rev = rev if res["success"]: html += "<tr %s>\n" % row_class else: html += "<tr class=failedrow>\n" for field in fields: val = res[field] # Trim the fractional seconds if field=="date": val = str(val)[0:19] # Add a trac link if field=="revision": try: num = int(val) except: num = 0 val = """<a href="http://trac.mantidproject.org/mantid/changeset/%d">%d</a>""" % (num, num) if field=="runtime": val = "%.3f" % (res["runtime"]/res["iterations"]) html += "<td>%s</td>" % val html += "\n</tr>\n" # # Add the row header every 30 entries # count += 1 # if count % 30 == 0: html += table_row_header # And one more at the end for good measure html += table_row_header html += "</table>" if len(results)> 0: html += """<h3>Environment</h3> %s""" % make_environment_html(results[0]) html += default_html_footer # last_date = sorted[-1][1]["date"] # results = get_results(name, type='', get_log=False, where_clause=" date = '%s'" % last_date) # if len(results)>0: # html += f = open(os.path.join(basedir, "%s.htm" % name), "w") f.write(html) f.close()
def make_detailed_html_file(basedir, name, fig1, fig2, last_num): """ Create a detailed HTML report for the named test """ html = DEFAULT_PLOTLY_HEADER html += """<h1>Detailed report for %s</h1><br>""" % (name) html += fig1 + "\n" html += fig2 + "\n" html += """<h3>Test Results</h3>""" fields = ['revision', 'date', 'commitid', 'compare', 'status', 'runtime', 'cpu_fraction', 'variables'] table_row_header = "<tr>" for field in fields: if field == "runtime": field = "Runtime/Iter." field = field[0].upper() + field[1:] table_row_header += "<th>%s</th>" % field table_row_header += "</tr>" html += """<table border="1">""" + table_row_header table_html = '' results = get_results(name, type='', where_clause='', orderby_clause="ORDER BY revision, variables, date") data = [(res["revision"], res["variables"], res["date"], res) for res in results] count = 0 last_rev = 0 commitid = '' last_commitid = '' row_class = '' table_rows = [] for (rev, variable, date, res) in data: table_row_html = '' if (rev != last_rev): # Changed SVN revision. Swap row color if row_class == '': row_class = "class=alternaterow" else: row_class = '' last_rev = rev if commitid != last_commitid: last_commitid = commitid if res["success"]: table_row_html += "<tr %s>\n" % row_class else: table_row_html += "<tr class=failedrow>\n" for field in fields: val = '' if field == 'compare': # Comparison to previous commit, if anything can be done if (last_commitid != ""): val = """<a href="%s/compare/%s...%s">diff</a>""" % ( MANTID_ADDRESS, last_commitid, commitid) else: # Normal fields val = res[field] # Trim the fractional seconds if field == "date": val = str(val)[0:19] # Add a trac link if field == "commitid": commitid = val partial_commitid = val if (len(partial_commitid) > 7): partial_commitid = partial_commitid[0:7]; val = """<a href="%s/commit/%s">%s</a>""" % (MANTID_ADDRESS, commitid, partial_commitid) if field == "runtime": val = "%.3f" % (res["runtime"]) table_row_html += "<td>%s</td>" % val table_row_html += "\n</tr>\n" table_rows.append(table_row_html) # Now print out all the rows in reverse order table_rows.reverse() for row in table_rows: html += row # And one more at the end for good measure html += table_row_header html += "</table>" if results: html += """<h3>Environment</h3> %s""" % make_environment_html(results[0]) html += DEFAULT_HTML_FOOTER f = open(os.path.join(basedir, "%s.htm" % name), "w") html = html.replace("\n", os.linesep) # Fix line endings for windows f.write(html) f.close()
def plot_runtime(annotate, saveImage, path, **kwargs): name = kwargs['name'] (xData, yData) = get_runtime_data(**kwargs) trace1 = go.Scatter(x=xData, y=yData, mode='lines+markers', marker=dict( size='5', color="blue" ) ) annotations = [] last_num = kwargs.get('last_num', None) if annotate and not saveImage: # retrieve commitids for annotation on the plotly graph results = get_results(name, orderby_clause='ORDER BY revision, variables, date') commitids = ["""<a href="%s/commit/%s"> </a>""" % (MANTID_ADDRESS, res["commitid"]) for res in results] if last_num is not None: commitids = commitids[-last_num:] for x, y, text in zip(xData, yData, commitids): annotations.append( dict( x=x, y=y, text=text, showarrow=False, font=dict(family='sans serif', size=10), xanchor='center', yanchor='center' ) ) if last_num is not None: title = "Runtime History of %s (last %d revs)" % (name, last_num) else: title = "Runtime History of %s (all revs)" % name yAxisTitle = 'Runtime/iteration (sec)' xAxisTitle = kwargs['x_field'] layout = go.Layout(showlegend=False, annotations=annotations, title=title, xaxis=dict(title=xAxisTitle), yaxis=dict( title=yAxisTitle, range=[0, np.amax(yData)] ) ) data = [trace1] fig = go.Figure(data=data, layout=layout) if saveImage: im_filename = name + ".png" plt.ioff() plt.figure() plt.title(title) plt.xlabel(xAxisTitle) plt.ylabel(yAxisTitle) plt.plot(xData, yData, "-b.") plt.ylim(ymin=0) plt.savefig(os.path.join(path, im_filename)) plt.close() return """<img src="%s"/>""" % im_filename else: return offline.plot(fig, output_type='div', show_link=False, auto_open=False, include_plotlyjs=False)