default="revision", help="Field to use as the x-axis. Default: 'revision'. Other possibilities: 'date'.") parser.add_argument('dbfile', metavar='DBFILE', type=str, nargs='+', default=["./MantidSystemTests.db"], help='Required: Path to the SQL database file(s).') args = parser.parse_args() # Import the manager definition import analysis import sqlresults if len(args.dbfile) > 1: # Several files - join them into one big .db dbfile = join_databases(args.dbfile) else: # Only one file - use it dbfile = args.dbfile[0] if not os.path.exists(dbfile): print "Error! Could not find", dbfile sys.exit(1) # This is where we look for the DB file sqlresults.set_database_filename(dbfile) # Make the report analysis.generate_html_report(args.path, 100, args.x_field)
# --------- Table with the summary of latest results -------- html += """<h2>Overall Results Summary</h2>""" html += get_html_summary_table(test_names) html += DEFAULT_HTML_FOOTER f = open(os.path.join(basedir, "report.htm"), "w") html = html.replace("\n", os.linesep) # Fix line endings for windows f.write(html) f.close() # -------- Overview of plots ------------ f = open(os.path.join(basedir, "overview_plot.htm"), "w") overview_html = overview_html.replace( "\n", os.linesep) # Fix line endings for windows f.write(overview_html) f.close() print "Report complete!" # ============================================================================================ if __name__ == "__main__": sqlresults.set_database_filename("MyFakeData.db") # Make up some test data if 0: if os.path.exists("MyFakeData.db"): os.remove("MyFakeData.db") sqlresults.generate_fake_data(300) generate_html_report("../Report", 50)
f.write(html) f.close() # -------- Overview of plots ------------ f = open(os.path.join(basedir, "overview_plot.htm"), "w") overview_html = overview_html.replace("\n", os.linesep) # Fix line endings for windows f.write(overview_html) f.close() print "Report complete!" #============================================================================================ if __name__ == "__main__": sqlresults.set_database_filename("MyFakeData.db") # Make up some test data if 0: if os.path.exists("MyFakeData.db"): os.remove("MyFakeData.db") sqlresults.generate_fake_data(300) generate_html_report("../Report", 50) # plot_runtime(name='MyFakeTest', x_field='revision') # plot_runtime(name='MyFakeTest', x_field='date') # plot_success_count() # show()
def run(args): """ Execute the program """ print print "=============== Checking For Performance Loss =====================" dbfile = args.db[0] if not os.path.exists(dbfile): print "Database file %s not found." sys.exit(1) # Set the database to the one given sqlresults.set_database_filename(dbfile); # Convert the arguments. Will throw if the user is stupid. avg = int(args.avg) tol = float(args.tol) rev = sqlresults.get_latest_revison() print "Comparing the average of the %d revisions before rev. %d. Tolerance of %g %%." % (avg, rev, tol) if args.verbose: print # For limiting the results limit = 50*avg; names = sqlresults.get_all_test_names("revision = %d" % rev) if len(names) == 0: print "Error! No tests found at revision number %d.\n" % rev sys.exit(1) bad_results = "" good_results = "" num_same = 0 num_good = 0 num_bad = 0 num_notenoughstats=0 # The timing resolution is different across platforms and the faster tests # can cause more false positives on the lower-resolution clocks. We'll # up the tolerance for those taking less time than 10ms. timer_resolution_hi = 0.01 timer_resolution_lo = 0.0011 regression_names = [] speedup_names = [] for name in names: (r, t) = analysis.get_runtime_data(name, x_field='revision') r = np.array(r) t = np.array(t) # this is the timing of the current revision current_time = t[r == rev] tolerance = tol if current_time < timer_resolution_hi: # Increase the tolerance to avoid false positives if current_time < timer_resolution_lo: # Very fast tests are twitchy tolerance = 100.0 else: tolerance = 70.0 # Cut out any times after or = to the current rev t = t[r < rev] # Keep the latest "avg" # t = t[len(t)-avg:] if (len(t) == avg): # This is the average baseline_time = np.mean(t) # % speed change pct = ((baseline_time / current_time) - 1) * 100 # Did we fail (slow down too much) if pct < -tolerance: regression_names.append(name) elif pct > tolerance: speedup_names.append(name) regLinks = ["http://builds.mantidproject.org/job/master_performancetests2/Master_branch_performance_tests/{}.htm".format(name) for name in regression_names] speedLinks = ["http://builds.mantidproject.org/job/master_performancetests2/Master_branch_performance_tests/{}.htm".format(name) for name in speedup_names] email = secureemail.SendEmailSecure(args.sender, args.pwd, args.recipient, regLinks, speedLinks) email.send()
'--commit', dest='commitid', default="", help='Commit ID of the current build (a 40-character SHA string).') parser.add_argument('xmlpath', metavar='XMLPATH', type=str, nargs='+', default="", help='Required: Path to the Xunit XML files.') args = parser.parse_args() # Setup the SQL database but only if it does not exist sqlresults.set_database_filename(args.db) if not os.path.exists(args.db): sqlresults.setup_database() # Set up the reporter sql_reporter = sqlresults.SQLResultReporter() variables = args.variables # Add a new revision and get the "revision" number revision = sqlresults.add_revision() # Save the commitid commitid = args.commitid # If a directory has been provided, look there for all of the XML files if os.path.isdir(args.xmlpath[0]): xmldir = args.xmlpath[0] if not os.path.isabs(xmldir):
parser.add_argument('--variables', dest='variables', default="", help='Optional string of comma-separated "VAR1NAME=VALUE,VAR2NAME=VALUE2" giving some parameters used, e.g. while building.') parser.add_argument('--commit', dest='commitid', default="", help='Commit ID of the current build (a 40-character SHA string).') parser.add_argument('xmlpath', metavar='XMLPATH', type=str, nargs='+', default="", help='Required: Path to the Xunit XML files.') args = parser.parse_args() # Setup the SQL database but only if it does not exist sqlresults.set_database_filename(args.db) if not os.path.exists(args.db): sqlresults.setup_database() # Set up the reporter sql_reporter = sqlresults.SQLResultReporter() variables = args.variables # Add a new revision and get the "revision" number revision = sqlresults.add_revision() # Save the commitid commitid = args.commitid # If a directory has been provided, look there for all of the XML files if os.path.isdir(args.xmlpath[0]): xmldir = args.xmlpath[0] if not os.path.isabs(xmldir):
def run(args): """ Execute the program """ print() print( "=============== Checking For Performance Loss =====================") dbfile = args.db[0] if not os.path.exists(dbfile): print("Database file %s not found.") sys.exit(1) # Set the database to the one given sqlresults.set_database_filename(dbfile) # Convert the arguments. Will throw if the user is stupid. avg = int(args.avg) tol = float(args.tol) rev = sqlresults.get_latest_revison() print( "Comparing the average of the %d revisions before rev. %d. Tolerance of %g %%." % (avg, rev, tol)) if args.verbose: print # For limiting the results limit = 50 * avg names = sqlresults.get_all_test_names("revision = %d" % rev) if len(names) == 0: print("Error! No tests found at revision number %d.\n" % rev) sys.exit(1) bad_results = "" good_results = "" num_same = 0 num_good = 0 num_bad = 0 num_notenoughstats = 0 # The timing resolution is different across platforms and the faster tests # can cause more false positives on the lower-resolution clocks. We'll # up the tolerance for those taking less time than 10ms. timer_resolution_hi = 0.01 timer_resolution_lo = 0.0011 regression_names = [] speedup_names = [] for name in names: (r, t) = analysis.get_runtime_data(name, x_field='revision') r = np.array(r) t = np.array(t) # this is the timing of the current revision current_time = t[r == rev] tolerance = tol if current_time < timer_resolution_hi: # Increase the tolerance to avoid false positives if current_time < timer_resolution_lo: # Very fast tests are twitchy tolerance = 100.0 else: tolerance = 70.0 # Cut out any times after or = to the current rev t = t[r < rev] # Keep the latest "avg" # t = t[len(t) - avg:] if (len(t) == avg): # This is the average baseline_time = np.mean(t) # % speed change pct = ((baseline_time / current_time) - 1) * 100 # Did we fail (slow down too much) if pct < -tolerance: regression_names.append(name) elif pct > tolerance: speedup_names.append(name) regLinks = [ "http://builds.mantidproject.org/job/master_performancetests2/Master_branch_performance_tests/{}.htm" .format(name) for name in regression_names ] speedLinks = [ "http://builds.mantidproject.org/job/master_performancetests2/Master_branch_performance_tests/{}.htm" .format(name) for name in speedup_names ] email = secureemail.SendEmailSecure(args.sender, args.pwd, args.recipient, regLinks, speedLinks) email.send()
def run(args): """ Execute the program """ print print "=============== Checking For Performance Loss =====================" dbfile = args.db[0] if not os.path.exists(dbfile): print "Database file %s not found." sys.exit(1) # Set the database to the one given sqlresults.set_database_filename(dbfile) # Convert the arguments. Will throw if the user is stupid. avg = int(args.avg) tol = float(args.tol) rev = sqlresults.get_latest_revison() print "Comparing the average of the %d revisions before rev. %d. Tolerance of %g %%." % ( avg, rev, tol) if args.verbose: print # For limiting the results limit = 50 * avg names = sqlresults.get_all_test_names("revision = %d" % rev) if len(names) == 0: print "Error! No tests found at revision number %d.\n" % rev sys.exit(1) bad_results = "" good_results = "" num_same = 0 num_good = 0 num_bad = 0 num_notenoughstats = 0 # The timing resolution is different across platforms and the faster tests # can cause more false positives on the lower-resolution clocks. We'll # up the tolerance for those taking less time than 10ms. timer_resolution_hi = 0.01 timer_resolution_lo = 0.0011 for name in names: (r, t) = analysis.get_runtime_data(name, x_field='revision') r = np.array(r) t = np.array(t) # this is the timing of the current revision current_time = t[r == rev] tolerance = tol if current_time < timer_resolution_hi: # Increase the tolerance to avoid false positives if current_time < timer_resolution_lo: # Very fast tests are twitchy tolerance = 100.0 else: tolerance = 70.0 print "%s is fast, tolerance has been increased to %f" % ( name, tolerance) # Cut out any times after or = to the current rev t = t[r < rev] # Keep the latest "avg" # t = t[len(t) - avg:] if (len(t) == avg): # This is the average baseline_time = np.mean(t) # % speed change pct = ((baseline_time / current_time) - 1) * 100 timing_str = "was %8.3f s, now %8.3f s. Speed changed by %+8.1f %%." % ( baseline_time, current_time, pct) if args.verbose: print "%s" % name print " %s" % timing_str # Did we fail (slow down too much) if pct < -tolerance: bad_results += "Warning! Slow down in performance test %s\n" % name bad_results += " (%s)\n" % timing_str num_bad += 1 # Hey you got better! elif pct > tolerance: good_results += "Congratulations! You sped up the performance of test %s\n" % name good_results += " (%s)\n" % timing_str num_good += 1 # No change else: num_same += 1 else: # Not enough stats num_notenoughstats += 1 if args.verbose: print "%s" % name print " Insufficient statistics." np.random.seed() def waswere(num): if num > 1 or num == 0: return "were" else: return "was" print print "-------- Summary ---------" print print "Out of %d tests, %d %s the same speed, %d %s faster, and %d %s slower." % ( len(names), num_same, waswere(num_same), num_good, waswere(num_good), num_bad, waswere(num_bad)) if (num_notenoughstats > 0): print "%d test(s) did not have a history of %d previous revisions and were not compared." % ( num_notenoughstats, avg) print if num_good > 0: print good_results if num_bad > 0: print bad_results quips = [ "Bad programmer! No cookie!", "Tsk. Tsk. Tsk.", "How could you!?", "Now get back in there and fix it!", "What did you do? WHAT DID YOU DO!?!" ] print quips[np.random.randint(len(quips))] print
def run(args): """ Execute the program """ print print "=============== Checking For Performance Loss =====================" dbfile = args.db[0] if not os.path.exists(dbfile): print "Database file %s not found." sys.exit(1) # Set the database to the one given sqlresults.set_database_filename(dbfile); # Convert the arguments. Will throw if the user is stupid. avg = int(args.avg) tol = float(args.tol) rev = int(args.revision[0]) print "Comparing the average of the %d revisions before rev. %d. Tolerance of %g %%." % (avg, rev, tol) if args.verbose: print # For limiting the results limit = 50*avg; names = sqlresults.get_all_test_names("revision = %d" % rev) if len(names) == 0: print "Error! No tests found at revision number %d.\n" % rev sys.exit(1) bad_results = "" good_results = "" num_same = 0 num_good = 0 num_bad = 0 num_notenoughstats=0 for name in names: (r, t) = analysis.get_runtime_data(name, x_field='revision') r = np.array(r) t = np.array(t) # this is the timing of the current revision current_time = t[r == rev] # Cut out any times after or = to the current rev t = t[r < rev] # Keep the latest "avg" # t = t[len(t)-avg:] if (len(t) == avg): # This is the average baseline_time = np.mean(t) # % speed change pct = ((baseline_time / current_time) - 1) * 100 timing_str = "was %8.3f s, now %8.3f s. Speed changed by %+8.1f %%." % (baseline_time, current_time, pct) if args.verbose: print "%s" % name print " %s" % timing_str # Did we fail (slow down too much) if pct < -tol: bad_results += "Warning! Slow down in performance test %s\n" % name bad_results += " (%s)\n" % timing_str num_bad += 1 # Hey you got better! elif pct > tol: good_results += "Congratulations! You sped up the performance of test %s\n" % name good_results += " (%s)\n" % timing_str num_good += 1 # No change else: num_same += 1 else: # Not enough stats num_notenoughstats += 1 if args.verbose: print "%s" % name print " Insufficient statistics." np.random.seed() def waswere(num): if num > 1 or num==0: return "were" else: return "was" print print "-------- Summary ---------" print print "Out of %d tests, %d %s the same speed, %d %s faster, and %d %s slower." % (len(names), num_same, waswere(num_same), num_good, waswere(num_good), num_bad, waswere(num_bad)) if (num_notenoughstats > 0): print "%d test(s) did not have a history of %d previous revisions and were not compared." % (num_notenoughstats, avg) print if num_good > 0: print good_results if num_bad > 0: print bad_results quips = ["Bad programmer! No cookie!", "Tsk. Tsk. Tsk.", "How could you!?", "Now get back in there and fix it!", "What did you do? WHAT DID YOU DO!?!"] print quips[np.random.randint(len(quips))] print