def send_to_graph(results_server, results_link, machine, date, browser_config, results): links = '' files = [] #construct all the files of data, one file per test and one file per counter for testname in results: vals = [] fullname = testname browser_dump, counter_dump = results[testname] utils.debug("Working with test: " + testname) utils.debug("Sending results: " + " ".join(browser_dump)) utils.stamped_msg("Generating results file: " + testname, "Started") if testname in ('ts', 'twinopen'): #non-tpformat results for bd in browser_dump: vals.extend([[x, 'NULL'] for x in bd.split('|')]) else: #tpformat results fullname += browser_config['test_name_extension'] for bd in browser_dump: bd.rstrip('\n') page_results = bd.splitlines() for line in page_results: val, page = process_tpformat(line) if val > -1: vals.append([val, page]) files.append( construct_file(machine, fullname, browser_config['branch_name'], browser_config['sourcestamp'], browser_config['buildid'], date, vals)) utils.stamped_msg("Generating results file: " + testname, "Stopped") for cd in counter_dump: for count_type in cd: vals = [[x, 'NULL'] for x in cd[count_type]] counterName = testname + '_' + shortName(count_type) if testname not in ('ts', 'twinopen'): counterName += browser_config['test_name_extension'] utils.stamped_msg("Generating results file: " + counterName, "Started") files.append( construct_file(machine, counterName, browser_config['branch_name'], browser_config['sourcestamp'], browser_config['buildid'], date, vals)) utils.stamped_msg("Generating results file: " + counterName, "Stopped") #send all the files along to the graph server for filename in files: links += process_Request( post_test_result(results_server, results_link, filename)) os.remove(filename) utils.stamped_msg("Transmitting test: " + testname, "Stopped") return links
test_name_extension=browser_config['test_name_extension']) # results links results_urls, results_options = configurator.output_options() talos_results.check_output_formats(results_urls, **results_options) # setup a webserver, if --develop is specified to PerfConfigurator.py httpd = None if browser_config['develop'] == True: httpd = setup_webserver(browser_config['webserver']) if httpd: httpd.start() # run the tests utils.startTimer() utils.stamped_msg(title, "Started") for test in tests: testname = test['name'] utils.stamped_msg("Running test " + testname, "Started") if os.path.exists('logcat.log'): os.unlink('logcat.log') try: mytest = TTest(browser_config['remote']) if mytest: talos_results.add(mytest.runTest(browser_config, test)) else: utils.stamped_msg("Error found while running %s" % testname, "Error") except talosRegression, tr: utils.stamped_msg("Detected a regression for " + testname, "Stopped")
def __call__(self): """ results to send to graphserver: construct all the strings of data, one string per test and one string per counter """ result_strings = [] info_dict = dict(title=self.results.title, date=self.results.date, branch_name=self.results.browser_config['branch_name'], sourcestamp=self.results.browser_config['sourcestamp'], buildid=self.results.browser_config['buildid'], browser_name=self.results.browser_config['browser_name'], browser_version=self.results.browser_config['browser_version'] ) for test in self.results.results: utils.debug("Working with test: %s", test.name()) # get full name of test testname = test.name() if test.format == 'tpformat': # for some reason, we append the test extension to tp results but not ts # http://hg.mozilla.org/build/talos/file/170c100911b6/talos/run_tests.py#l176 testname += self.results.test_name_extension utils.stamped_msg("Generating results file: %s" % test.name(), "Started") # HACK: when running xperf, we upload xperf counters to the graph server but we do not want to # upload the test results as they will confuse the graph server if not test.using_xperf: vals = [] for result in test.results: # per test filters _filters = self.results.filters if 'filters' in test.test_config: try: _filters = filter.filters_args(test.test_config['filters']) except AssertionError, e: raise utils.talosError(str(e)) vals.extend(result.values(_filters)) result_strings.append(self.construct_results(vals, testname=testname, **info_dict)) utils.stamped_msg("Generating results file: %s" % test.name(), "Stopped") # counter results for cd in test.all_counter_results: for counter_type, values in cd.items(): # get the counter name counterName = '%s_%s' % (test.name() , self.shortName(counter_type)) if not values: # failed to collect any data for this counter utils.stamped_msg("No results collected for: " + counterName, "Error") # NOTE: we are not going to enforce this warning for now as this happens too frequently: bugs 803413, 802475, 805925 # raise utils.talosError("Unable to proceed with missing counter '%s'" % counterName) # (jhammel: we probably should do this in e.g. results.py vs in graphserver-specific code anyway) # exclude counters whose values are tuples (bad for graphserver) if len(values) > 0 and isinstance(values[0], list): continue # counter values vals = [[x, 'NULL'] for x in values] # append test name extension but only for tpformat tests if test.format == 'tpformat': counterName += self.results.test_name_extension info = info_dict.copy() info['testname'] = counterName # append the counter string utils.stamped_msg("Generating results file: %s" % counterName, "Started") result_strings.append(self.construct_results(vals, **info)) utils.stamped_msg("Generating results file: %s" % counterName, "Stopped")
def __call__(self): """ results to send to graphserver: construct all the strings of data, one string per test and one string per counter """ result_strings = [] info_dict = dict( title=self.results.title, date=self.results.date, branch_name=self.results.browser_config['branch_name'], sourcestamp=self.results.browser_config['sourcestamp'], buildid=self.results.browser_config['buildid'], browser_name=self.results.browser_config['browser_name'], browser_version=self.results.browser_config['browser_version']) for test in self.results.results: logging.debug("Working with test: %s", test.name()) # get full name of test testname = test.name() if test.format == 'tpformat': # for some reason, we append the test extension to tp results # but not ts # http://hg.mozilla.org/build/talos/file/170c100911b6/talos # /run_tests.py#l176 testname += test.extension() logging.debug("Generating results file: %s" % test.name()) # HACK: when running xperf, we upload xperf counters to the graph # server but we do not want to # upload the test results as they will confuse the graph server if not (test.format == 'tpformat' and test.using_xperf): vals = [] for result in test.results: filtered_val = result.values(testname, test.test_config['filters']) vals.extend([[i['filtered'], j] for i, j in filtered_val]) result_strings.append( self.construct_results(vals, testname=testname, **info_dict)) # counter results for cd in test.all_counter_results: for counter_type, values in cd.items(): # get the counter name counterName = '%s_%s' % (test.name(), self.shortName(counter_type)) if not values: # failed to collect any data for this counter utils.stamped_msg( "No results collected for: " + counterName, "Error") # NOTE: we are not going to enforce this warning for now as this happens too # frequently: bugs 803413, 802475, 805925 # raise utils.TalosError("Unable to proceed with missing # counter '%s'" % counterName) # (jhammel: we probably should do this in e.g. results.py vs in # graphserver-specific code anyway) # exclude counters whose values are tuples (bad for # graphserver) if len(values) > 0 and isinstance(values[0], list): print "Not uploading counter data for %s" % counterName print values continue if test.mainthread() and 'mainthreadio' in counterName: print("Not uploading Mainthread IO data for %s" % counterName) print values continue # counter values vals = [[x, 'NULL'] for x in values] # append test name extension but only for tpformat tests if test.format == 'tpformat': counterName += test.extension() info = info_dict.copy() info['testname'] = counterName # append the counter string utils.stamped_msg( "Generating results file: %s" % counterName, "Started") result_strings.append(self.construct_results(vals, **info)) utils.stamped_msg( "Generating results file: %s" % counterName, "Stopped") return result_strings
# results links results_urls, results_options = configurator.output_options() talos_results.check_output_formats(results_urls, **results_options) results_log = browser_config['results_log'] # setup a webserver, if --develop is specified to PerfConfigurator.py httpd = None if browser_config['develop'] == True: httpd = setup_webserver(browser_config['webserver']) if httpd: httpd.start() # run the tests utils.startTimer() utils.stamped_msg(title, "Started") for test in tests: testname = test['name'] test['browser_log'] = browser_config['browser_log'] utils.stamped_msg("Running test " + testname, "Started") if os.path.exists('logcat.log'): os.unlink('logcat.log') try: mytest = TTest(browser_config['remote']) if mytest: talos_results.add(mytest.runTest(browser_config, test)) else: utils.stamped_msg("Error found while running %s" % testname, "Error")
def test_file(filename): """Runs the Ts and Tp tests on the given config file and generates a report. Args: filename: the name of the file to run the tests on """ browser_config = [] tests = [] title = '' testdate = '' csv_dir = '' results_server = '' results_link = '' results = {} # Read in the profile info from the YAML config file config_file = open(filename, 'r') yaml_config = yaml.load(config_file) config_file.close() for item in yaml_config: if item == 'title': title = yaml_config[item] elif item == 'testdate': testdate = yaml_config[item] elif item == 'csv_dir': csv_dir = os.path.normpath(yaml_config[item]) if not os.path.exists(csv_dir): print "FAIL: path \"" + csv_dir + "\" does not exist" sys.exit(0) elif item == 'results_server': results_server = yaml_config[item] elif item == 'results_link' : results_link = yaml_config[item] if (results_link != results_server != ''): if not post_file.link_exists(results_server, results_link): sys.exit(0) browser_config = {'preferences' : yaml_config['preferences'], 'extensions' : yaml_config['extensions'], 'firefox' : yaml_config['firefox'], 'branch' : yaml_config['branch'], 'buildid' : yaml_config['buildid'], 'profile_path' : yaml_config['profile_path'], 'env' : yaml_config['env'], 'dirs' : yaml_config['dirs'], 'init_url' : yaml_config['init_url']} #normalize paths to work accross platforms browser_config['firefox'] = os.path.normpath(browser_config['firefox']) if browser_config['profile_path'] != {}: browser_config['profile_path'] = os.path.normpath(browser_config['profile_path']) for dir in browser_config['dirs']: browser_config['dirs'][dir] = os.path.normpath(browser_config['dirs'][dir]) tests = yaml_config['tests'] config_file.close() if (testdate != ''): date = int(time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT'))) else: date = int(time.time()) #TODO get this into own file utils.debug("using testdate: %d" % date) utils.debug("actual date: %d" % int(time.time())) utils.stamped_msg(title, "Started") for test in tests: utils.stamped_msg("Running test " + test, "Started") try: browser_dump, counter_dump = ttest.runTest(browser_config, tests[test]) except talosError, e: utils.stamped_msg("Failed " + test, "Stopped") print 'FAIL: Busted: ' + test print 'FAIL: ' + e.msg sys.exit(0) utils.debug("Received test results: " + " ".join(browser_dump)) results[test] = [browser_dump, counter_dump] utils.stamped_msg("Completed test " + test, "Stopped")
def test_file(filename): """Runs the talos tests on the given config file and generates a report. Args: filename: the name of the file to run the tests on """ browser_config = [] tests = [] title = '' testdate = '' csv_dir = '' results_server = '' results_link = '' old_results_server = '' old_results_link = '' results = {} # Read in the profile info from the YAML config file config_file = open(filename, 'r') yaml_config = yaml.load(config_file) config_file.close() for item in yaml_config: if item == 'title': title = yaml_config[item] elif item == 'testdate': testdate = yaml_config[item] elif item == 'csv_dir': csv_dir = os.path.normpath(yaml_config[item]) if not os.path.exists(csv_dir): print "FAIL: path \"" + csv_dir + "\" does not exist" sys.exit(0) elif item == 'results_server': results_server = yaml_config[item] elif item == 'results_link': results_link = yaml_config[item] elif item == 'old_results_server': old_results_server = yaml_config[item] elif item == 'old_results_link': old_results_link = yaml_config[item] if (results_link != results_server != ''): if not post_file.link_exists(results_server, results_link): sys.exit(0) if (old_results_link != old_results_server != ''): if not post_file.link_exists(old_results_server, old_results_link): sys.exit(0) browser_config = { 'preferences': yaml_config['preferences'], 'extensions': yaml_config['extensions'], 'browser_path': yaml_config['browser_path'], 'browser_log': yaml_config['browser_log'], 'symbols_path': yaml_config.get('symbols_path', None), 'browser_wait': yaml_config['browser_wait'], 'process': yaml_config['process'], 'extra_args': yaml_config['extra_args'], 'branch': yaml_config['branch'], 'buildid': yaml_config['buildid'], 'profile_path': yaml_config['profile_path'], 'env': yaml_config['env'], 'dirs': yaml_config['dirs'], 'init_url': yaml_config['init_url'] } if 'branch_name' in yaml_config: browser_config['branch_name'] = yaml_config['branch_name'] if 'test_name_extension' in yaml_config: browser_config['test_name_extension'] = yaml_config[ 'test_name_extension'] else: browser_config['test_name_extension'] = '' #normalize paths to work accross platforms browser_config['browser_path'] = os.path.normpath( browser_config['browser_path']) if browser_config['profile_path'] != {}: browser_config['profile_path'] = os.path.normpath( browser_config['profile_path']) for dir in browser_config['dirs']: browser_config['dirs'][dir] = os.path.normpath( browser_config['dirs'][dir]) tests = yaml_config['tests'] config_file.close() if (testdate != ''): date = int( time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT'))) else: date = int(time.time()) #TODO get this into own file utils.debug("using testdate: %d" % date) utils.debug("actual date: %d" % int(time.time())) print 'RETURN:s: %s' % title #pull buildid & sourcestamp from browser browser_config = browserInfo(browser_config) utils.startTimer() utils.stamped_msg(title, "Started") for test in tests: testname = test['name'] utils.stamped_msg("Running test " + testname, "Started") try: browser_dump, counter_dump = ttest.runTest(browser_config, test) utils.debug("Received test results: " + " ".join(browser_dump)) results[testname] = [browser_dump, counter_dump] # If we're doing CSV, write this test immediately (bug 419367) if csv_dir != '': send_to_csv(csv_dir, {testname: results[testname]}) except talosError, e: utils.stamped_msg("Failed " + testname, "Stopped") print 'FAIL: Busted: ' + testname print 'FAIL: ' + e.msg utils.stamped_msg("Completed test " + testname, "Stopped")
def send_to_graph(results_server, results_link, machine, date, browser_config, results, amo): links = '' result_strings = [] result_testnames = [] #construct all the strings of data, one string per test and one string per counter for testname in results: vals = [] fullname = testname browser_dump, counter_dump, print_format = results[testname] utils.debug("Working with test: " + testname) utils.debug("Sending results: " + " ".join(browser_dump)) utils.stamped_msg("Generating results file: " + testname, "Started") if print_format == 'tsformat': #non-tpformat results for bd in browser_dump: vals.extend([[x, 'NULL'] for x in bd.split('|')]) elif print_format == 'tpformat': #tpformat results fullname += browser_config['test_name_extension'] for bd in browser_dump: bd.rstrip('\n') page_results = bd.splitlines() for line in page_results: val, page = process_tpformat(line) if val > -1 : vals.append([val, page]) else: raise talosError("Unknown print format in send_to_graph") result_strings.append(construct_results(machine, fullname, browser_config, date, vals, amo)) result_testnames.append(fullname) utils.stamped_msg("Generating results file: " + testname, "Stopped") #counters collected for this test for cd in counter_dump: for count_type in cd: counterName = testname + '_' + shortName(count_type) if cd[count_type] == []: #failed to collect any data for this counter utils.stamped_msg("No results collected for: " + counterName, "Error") continue vals = [[x, 'NULL'] for x in cd[count_type]] if print_format == "tpformat": counterName += browser_config['test_name_extension'] utils.stamped_msg("Generating results file: " + counterName, "Started") result_strings.append(construct_results(machine, counterName, browser_config, date, vals, amo)) result_testnames.append(counterName) utils.stamped_msg("Generating results file: " + counterName, "Stopped") #send all the strings along to the graph server for data_string, testname in zip(result_strings, result_testnames): RETRIES = 5 wait_time = 5 times = 0 msg = "" while (times < RETRIES): try: utils.stamped_msg("Transmitting test: " + testname, "Started") links += process_Request(post_file.post_multipart(results_server, results_link, [("key", "value")], [("filename", "data_string", data_string)])) break except talosError, e: msg = e.msg except Exception, e: msg = str(e) times += 1 time.sleep(wait_time) wait_time += wait_time
def test_file(filename): """Runs the Ts and Tp tests on the given config file and generates a report. Args: filename: the name of the file to run the tests on """ browser_config = [] tests = [] title = '' testdate = '' csv_dir = '' results_server = '' results_link = '' results = {} # Read in the profile info from the YAML config file config_file = open(filename, 'r') yaml_config = yaml.load(config_file) config_file.close() for item in yaml_config: if item == 'title': title = yaml_config[item] elif item == 'testdate': testdate = yaml_config[item] elif item == 'csv_dir': csv_dir = os.path.normpath(yaml_config[item]) if not os.path.exists(csv_dir): print "FAIL: path \"" + csv_dir + "\" does not exist" sys.exit(0) elif item == 'results_server': results_server = yaml_config[item] elif item == 'results_link': results_link = yaml_config[item] if (results_link != results_server != ''): if not post_file.link_exists(results_server, results_link): sys.exit(0) browser_config = { 'preferences': yaml_config['preferences'], 'extensions': yaml_config['extensions'], 'firefox': yaml_config['firefox'], 'branch': yaml_config['branch'], 'buildid': yaml_config['buildid'], 'profile_path': yaml_config['profile_path'], 'env': yaml_config['env'], 'dirs': yaml_config['dirs'], 'init_url': yaml_config['init_url'] } #normalize paths to work accross platforms browser_config['firefox'] = os.path.normpath(browser_config['firefox']) if browser_config['profile_path'] != {}: browser_config['profile_path'] = os.path.normpath( browser_config['profile_path']) for dir in browser_config['dirs']: browser_config['dirs'][dir] = os.path.normpath( browser_config['dirs'][dir]) tests = yaml_config['tests'] config_file.close() if (testdate != ''): date = int( time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT'))) else: date = int(time.time()) #TODO get this into own file utils.debug("using testdate: %d" % date) utils.debug("actual date: %d" % int(time.time())) utils.stamped_msg(title, "Started") for test in tests: utils.stamped_msg("Running test " + test, "Started") try: browser_dump, counter_dump = ttest.runTest(browser_config, tests[test]) except talosError, e: utils.stamped_msg("Failed " + test, "Stopped") print 'FAIL: Busted: ' + test print 'FAIL: ' + e.msg sys.exit(0) utils.debug("Received test results: " + " ".join(browser_dump)) results[test] = [browser_dump, counter_dump] utils.stamped_msg("Completed test " + test, "Stopped")
def send_to_graph(results_server, results_link, title, date, browser_config, results): tbox = title url_format = "http://%s/%s" link_format = "<a href=\"%s\">%s</a>" #value, testname, tbox, timeval, date, branch, buildid, type, data result_format = "%.2f,%s,%s,%d,%d,%s,%s,%s,%s,\n" result_format2 = "%.2f,%s,%s,%d,%d,%s,%s,%s,\n" links = '' for res in results: browser_dump, counter_dump = results[res] utils.debug("Working with test: " + res) utils.debug("Sending results: " + " ".join(browser_dump)) utils.stamped_msg("Transmitting test: " + res, "Started") filename = tempfile.mktemp() tmpf = open(filename, "w") if res in ('ts', 'twinopen'): i = 0 for val in browser_dump: val_list = val.split('|') for v in val_list: tmpf.write(result_format % (float(v), res, tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete", "ms")) i += 1 else: # each line of the string is of the format i;page_name;median;mean;min;max;time vals\n name = '' if ((res == 'tp') or (res == 'tp_js')): name = '_loadtime' for bd in browser_dump: bd.rstrip('\n') page_results = bd.splitlines() i = 0 for mypage in page_results: r = mypage.split(';') #skip this line if it isn't the correct format if len(r) == 1: continue r[1] = r[1].rstrip('/') if r[1].find('/') > -1: page = r[1].split('/')[1] else: page = r[1] try: val = float(r[2]) except ValueError: print 'WARNING: value error for median in tp' val = 0 tmpf.write(result_format % (val, res + name, tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete", page)) i += 1 tmpf.flush() tmpf.close() links += post_chunk(results_server, results_link, res, filename) os.remove(filename) for cd in counter_dump: for count_type in cd: val_list = cd[count_type] chunks = chunk_list(val_list) chunk_link = '' i = 0 for chunk in chunks: filename = tempfile.mktemp() tmpf = open(filename, "w") for val in chunk: tmpf.write(result_format2 % (float(val), res + "_" + count_type.replace("%", "Percent"), tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete")) i += 1 tmpf.flush() tmpf.close() chunk_link = post_chunk( results_server, results_link, '%s_%s (%d values)' % (res, count_type, len(chunk)), filename) os.remove(filename) links += chunk_link utils.stamped_msg("Transmitting test: " + res, "Stopped") first_results = '' last_results = '' full_results = '\nRETURN:<p style="font-size:smaller;">Details:<br>' lines = links.split('\n') for line in lines: if line == "": continue values = line.split(":") linkName = values[1] if linkName in ('tp_pbytes', 'tp_%cpu'): continue if float(values[2]) > 0: linkName += ": " + str(values[2]) url = url_format % (results_server, values[0]) link = link_format % (url, linkName) first_results = first_results + "\nRETURN:" + link + "<br>" else: url = url_format % (results_server, values[0]) link = link_format % (url, linkName) last_results = last_results + '| ' + link + ' ' full_results = first_results + full_results + last_results + '|</p>' print full_results
utils.stamped_msg(title, "Started") for test in tests: utils.stamped_msg("Running test " + test, "Started") try: browser_dump, counter_dump = ttest.runTest(browser_config, tests[test]) except talosError, e: utils.stamped_msg("Failed " + test, "Stopped") print 'FAIL: Busted: ' + test print 'FAIL: ' + e.msg sys.exit(0) utils.debug("Received test results: " + " ".join(browser_dump)) results[test] = [browser_dump, counter_dump] utils.stamped_msg("Completed test " + test, "Stopped") utils.stamped_msg(title, "Stopped") #process the results if (results_server != '') and (results_link != ''): #send results to the graph server utils.stamped_msg("Sending results", "Started") send_to_graph(results_server, results_link, title, date, browser_config, results) utils.stamped_msg("Completed sending results", "Stopped") if csv_dir != '': send_to_csv(csv_dir, results) if __name__ == '__main__': optlist, args = getopt.getopt(sys.argv[1:], 'dn', ['debug', 'noisy']) for o, a in optlist:
def test_file(filename): """Runs the talos tests on the given config file and generates a report. Args: filename: the name of the file to run the tests on """ browser_config = [] tests = [] title = '' testdate = '' csv_dir = '' results_server = '' results_link = '' results = {} # Read in the profile info from the YAML config file config_file = open(filename, 'r') yaml_config = yaml.load(config_file) config_file.close() for item in yaml_config: if item == 'title': title = yaml_config[item] elif item == 'testdate': testdate = yaml_config[item] elif item == 'csv_dir': csv_dir = os.path.normpath(yaml_config[item]) if not os.path.exists(csv_dir): print "FAIL: path \"" + csv_dir + "\" does not exist" sys.exit(0) elif item == 'results_server': results_server = yaml_config[item] elif item == 'results_link' : results_link = yaml_config[item] if (results_link != results_server != ''): if not post_file.link_exists(results_server, results_link): sys.exit(0) browser_config = {'preferences' : yaml_config['preferences'], 'extensions' : yaml_config['extensions'], 'browser_path' : yaml_config['browser_path'], 'browser_log' : yaml_config['browser_log'], 'symbols_path' : yaml_config.get('symbols_path', None), 'browser_wait' : yaml_config['browser_wait'], 'process' : yaml_config['process'], 'extra_args' : yaml_config['extra_args'], 'branch' : yaml_config['branch'], 'buildid' : yaml_config['buildid'], 'env' : yaml_config['env'], 'dirs' : yaml_config['dirs'], 'init_url' : yaml_config['init_url']} if 'branch_name' in yaml_config: browser_config['branch_name'] = yaml_config['branch_name'] if 'test_name_extension' in yaml_config: browser_config['test_name_extension'] = yaml_config['test_name_extension'] else: browser_config['test_name_extension'] = '' #normalize paths to work accross platforms browser_config['browser_path'] = os.path.normpath(browser_config['browser_path']) for dir in browser_config['dirs']: browser_config['dirs'][dir] = os.path.normpath(browser_config['dirs'][dir]) tests = yaml_config['tests'] config_file.close() if (testdate != ''): date = int(time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT'))) else: date = int(time.time()) #TODO get this into own file utils.debug("using testdate: %d" % date) utils.debug("actual date: %d" % int(time.time())) print 'RETURN:s: %s' % title #pull buildid & sourcestamp from browser browser_config = browserInfo(browser_config) if ffprocess.checkAllProcesses(browser_config['process']): print "FAIL: all firefox processes must be closed before tests can be run" sys.exit(0) utils.startTimer() utils.stamped_msg(title, "Started") for test in tests: testname = test['name'] utils.stamped_msg("Running test " + testname, "Started") try: browser_dump, counter_dump, print_format = ttest.runTest(browser_config, test) utils.debug("Received test results: " + " ".join(browser_dump)) results[testname] = [browser_dump, counter_dump, print_format] # If we're doing CSV, write this test immediately (bug 419367) if csv_dir != '': send_to_csv(csv_dir, {testname : results[testname]}) except talosError, e: utils.stamped_msg("Failed " + testname, "Stopped") print 'FAIL: Busted: ' + testname print 'FAIL: ' + e.msg.replace('\n','\nRETURN:') utils.stamped_msg("Completed test " + testname, "Stopped")
def test_file(filename, to_screen): """Runs the talos tests on the given config file and generates a report. Args: filename: the name of the file to run the tests on to_screen: boolean, determine if all results should be outputed directly to stdout """ browser_config = [] tests = [] title = '' testdate = '' csv_dir = '' results_server = '' results_link = '' results = {} # Read in the profile info from the YAML config file config_file = open(filename, 'r') yaml_config = yaml.load(config_file) config_file.close() for item in yaml_config: if item == 'title': title = yaml_config[item] elif item == 'testdate': testdate = yaml_config[item] elif item == 'csv_dir': csv_dir = os.path.normpath(yaml_config[item]) if not os.path.exists(csv_dir): print "FAIL: path \"" + csv_dir + "\" does not exist" sys.exit(0) elif item == 'results_server': results_server = yaml_config[item] elif item == 'results_link': results_link = yaml_config[item] if (results_link != results_server != ''): if not post_file.link_exists(results_server, results_link): sys.exit(0) browser_config = { 'preferences': yaml_config['preferences'], 'extensions': yaml_config['extensions'], 'browser_path': yaml_config['browser_path'], 'browser_log': yaml_config['browser_log'], 'symbols_path': yaml_config.get('symbols_path', None), 'browser_wait': yaml_config['browser_wait'], 'process': yaml_config['process'], 'extra_args': yaml_config['extra_args'], 'branch': yaml_config['branch'], 'buildid': yaml_config['buildid'], 'env': yaml_config['env'], 'dirs': yaml_config.get('dirs', {}), 'bundles': yaml_config.get('bundles', {}), 'init_url': yaml_config['init_url'] } if 'child_process' in yaml_config: browser_config['child_process'] = yaml_config['child_process'] else: browser_config['child_process'] = 'plugin-container' if 'branch_name' in yaml_config: browser_config['branch_name'] = yaml_config['branch_name'] if 'test_name_extension' in yaml_config: browser_config['test_name_extension'] = yaml_config[ 'test_name_extension'] else: browser_config['test_name_extension'] = '' if 'sourcestamp' in yaml_config: browser_config['sourcestamp'] = yaml_config['sourcestamp'] if 'repository' in yaml_config: browser_config['repository'] = yaml_config['repository'] if 'deviceip' in yaml_config: browser_config['host'] = yaml_config['deviceip'] else: browser_config['host'] = '' if 'deviceport' in yaml_config: browser_config['port'] = yaml_config['deviceport'] else: browser_config['port'] = '' if 'webserver' in yaml_config: browser_config['webserver'] = yaml_config['webserver'] else: browser_config['webserver'] = '' if 'deviceroot' in yaml_config: browser_config['deviceroot'] = yaml_config['deviceroot'] else: browser_config['deviceroot'] = '' if 'remote' in yaml_config: browser_config['remote'] = yaml_config['remote'] else: browser_config['remote'] = False #normalize paths to work accross platforms dm = None if (browser_config['remote'] == True): import devicemanager dm = devicemanager.DeviceManager(browser_config['host'], browser_config['port']) browser_config['browser_path'] = os.path.normpath( browser_config['browser_path']) for dir in browser_config['dirs']: browser_config['dirs'][dir] = os.path.normpath( browser_config['dirs'][dir]) for bname in browser_config['bundles']: browser_config['bundles'][bname] = os.path.normpath( browser_config['bundles'][bname]) tests = yaml_config['tests'] config_file.close() if (testdate != ''): date = int( time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT'))) else: date = int(time.time()) #TODO get this into own file utils.debug("using testdate: %d" % date) utils.debug("actual date: %d" % int(time.time())) print 'RETURN:s: %s' % title #pull buildid & sourcestamp from browser browser_config = browserInfo(browser_config, devicemanager=dm) utils.startTimer() utils.stamped_msg(title, "Started") for test in tests: testname = test['name'] utils.stamped_msg("Running test " + testname, "Started") try: mytest = TTest(browser_config['remote']) browser_dump, counter_dump, print_format = mytest.runTest( browser_config, test) utils.debug("Received test results: " + " ".join(browser_dump)) results[testname] = [browser_dump, counter_dump, print_format] # If we're doing CSV, write this test immediately (bug 419367) if csv_dir != '': send_to_csv(csv_dir, {testname: results[testname]}) if to_screen: send_to_csv(None, {testname: results[testname]}) except talosError, e: utils.stamped_msg("Failed " + testname, "Stopped") print 'FAIL: Busted: ' + testname print 'FAIL: ' + e.msg.replace('\n', '\nRETURN:') utils.stamped_msg("Completed test " + testname, "Stopped")
def send_to_graph(results_server, results_link, machine, date, browser_config, results): links = '' result_strings = [] #construct all the strings of data, one string per test and one string per counter for testname in results: vals = [] fullname = testname browser_dump, counter_dump, print_format = results[testname] utils.debug("Working with test: " + testname) utils.debug("Sending results: " + " ".join(browser_dump)) utils.stamped_msg("Generating results file: " + testname, "Started") if print_format == 'tsformat': #non-tpformat results for bd in browser_dump: vals.extend([[x, 'NULL'] for x in bd.split('|')]) elif print_format == 'tpformat': #tpformat results fullname += browser_config['test_name_extension'] for bd in browser_dump: bd.rstrip('\n') page_results = bd.splitlines() for line in page_results: val, page = process_tpformat(line) if val > -1: vals.append([val, page]) else: raise talosError("Unknown print format in send_to_graph") result_strings.append( construct_results(machine, fullname, browser_config['branch_name'], browser_config['sourcestamp'], browser_config['buildid'], date, vals)) utils.stamped_msg("Generating results file: " + testname, "Stopped") #counters collected for this test for cd in counter_dump: for count_type in cd: vals = [[x, 'NULL'] for x in cd[count_type]] counterName = testname + '_' + shortName(count_type) utils.stamped_msg("Generating results file: " + counterName, "Started") if print_format == "tpformat": counterName += browser_config['test_name_extension'] utils.stamped_msg("Generating results file: " + counterName, "Started") result_strings.append( construct_results(machine, counterName, browser_config['branch_name'], browser_config['sourcestamp'], browser_config['buildid'], date, vals)) utils.stamped_msg("Generating results file: " + counterName, "Stopped") #send all the strings along to the graph server for data_string in result_strings: RETRIES = 5 wait_time = 5 times = 0 msg = "" while (times < RETRIES): try: utils.stamped_msg("Transmitting test: " + testname, "Started") links += process_Request( post_file.post_multipart( results_server, results_link, [("key", "value")], [("filename", "data_string", data_string)])) break except talosError, e: times += 1 msg = e.msg time.sleep(wait_time) wait_time += wait_time if times == RETRIES: raise talosError("Failed to send data %d times... quitting\n%s" % (RETRIES, msg)) utils.stamped_msg("Transmitting test: " + testname, "Stopped")
msg = "" while (times < RETRIES): try: utils.stamped_msg("Transmitting test: " + testname, "Started") links += process_Request(post_file.post_multipart(results_server, results_link, [("key", "value")], [("filename", "data_string", data_string)])) break except talosError, e: msg = e.msg except Exception, e: msg = str(e) times += 1 time.sleep(wait_time) wait_time += wait_time if times == RETRIES: raise talosError("Graph server unreachable (%d attempts)\n%s" % (RETRIES, msg)) utils.stamped_msg("Transmitting test: " + testname, "Stopped") return links def results_from_graph(links, results_server, amo): if amo: #only get a pass/fail back from the graph server lines = links.split('\n') for line in lines: if line == "": continue if line.lower() in ('success',): print 'RETURN:addon results inserted successfully' else: #take the results from the graph server collection script and put it into a pretty format for the waterfall
def send_to_csv(csv_dir, results): import csv def avg_excluding_max(val_list): """return float rounded to two decimal places, converted to string calculates the average value in the list exluding the max value""" i = len(val_list) total = sum(float(v) for v in val_list) maxval = max(float(v) for v in val_list) if total > maxval: avg = str(round((total - maxval)/(i-1), 2)) else: avg = str(round(total, 2)) return avg for res in results: browser_dump, counter_dump, print_format = results[res] if csv_dir: writer = csv.writer(open(os.path.join(csv_dir, res + '.csv'), "wb")) else: #working with stdout writer = csv.writer(sys.stdout) if print_format == 'tsformat': i = 0 res_list = [] writer.writerow(['i', 'val']) for val in browser_dump: val_list = val.split('|') for v in val_list: writer.writerow([i, v]) i += 1 res_list.append(v) writer.writerow(['RETURN: ' + res + ': ' + avg_excluding_max(res_list),]) elif print_format == 'tpformat': writer.writerow(['i', 'page', 'median', 'mean', 'min' , 'max', 'runs']) for bd in browser_dump: bd.rstrip('\n') page_results = bd.splitlines() i = 0 res_list = [] for mypage in page_results: r = mypage.split(';') #skip this line if it isn't the correct format if len(r) == 1: continue r[1] = r[1].rstrip('/') if r[1].find('/') > -1 : page = r[1].split('/')[1] else: page = r[1] res_list.append(r[2]) writer.writerow([i, page, r[2], r[3], r[4], r[5], '|'.join(r[6:])]) i += 1 writer.writerow(['RETURN: ' + res + ': ' + avg_excluding_max(res_list), ]) else: raise talosError("Unknown print format in send_to_csv") for cd in counter_dump: for count_type in cd: counterName = res + '_' + shortName(count_type) if cd[count_type] == []: #failed to collect any data for this counter utils.stamped_msg("No results collected for: " + counterName, "Error") continue if csv_dir: writer = csv.writer(open(os.path.join(csv_dir, counterName + '.csv'), "wb")) else: writer = csv.writer(sys.stdout) writer.writerow(['i', 'value']) i = 0 for val in cd[count_type]: writer.writerow([i, val]) i += 1 if isMemoryMetric(shortName(count_type)): writer.writerow(['RETURN: ' + counterName + ': ' + filesizeformat(avg_excluding_max(cd[count_type])),]) elif count_type == 'responsiveness': writer.writerow(['RETURN: ' + counterName + ': ' + responsiveness_Metric(cd[count_type]),]) else: writer.writerow(['RETURN: ' + counterName + ': ' + avg_excluding_max(cd[count_type]),])
utils.debug("actual date: %d" % int(time.time())) utils.stamped_msg(title, "Started") for test in tests: utils.stamped_msg("Running test " + test, "Started") try: browser_dump, counter_dump = ttest.runTest(browser_config, tests[test]) except talosError, e: utils.stamped_msg("Failed " + test, "Stopped") print 'FAIL: Busted: ' + test print 'FAIL: ' + e.msg sys.exit(0) utils.debug("Received test results: " + " ".join(browser_dump)) results[test] = [browser_dump, counter_dump] utils.stamped_msg("Completed test " + test, "Stopped") utils.stamped_msg(title, "Stopped") #process the results if (results_server != '') and (results_link != ''): #send results to the graph server utils.stamped_msg("Sending results", "Started") send_to_graph(results_server, results_link, title, date, browser_config, results) utils.stamped_msg("Completed sending results", "Stopped") if csv_dir != '': send_to_csv(csv_dir, results) if __name__=='__main__': optlist, args = getopt.getopt(sys.argv[1:], 'dn', ['debug', 'noisy']) for o, a in optlist: if o in ('-d', "--debug"): print 'setting debug'
def test_file(filename, to_screen, amo): """Runs the talos tests on the given config file and generates a report. Args: filename: the name of the file to run the tests on to_screen: boolean, determine if all results should be outputed directly to stdout """ browser_config = [] tests = [] title = '' testdate = '' csv_dir = '' results_server = '' results_link = '' results = {} # Read in the profile info from the YAML config file config_file = open(filename, 'r') yaml_config = yaml.load(config_file) config_file.close() for item in yaml_config: if item == 'title': title = yaml_config[item] elif item == 'testdate': testdate = yaml_config[item] elif item == 'csv_dir': csv_dir = os.path.normpath(yaml_config[item]) if not os.path.exists(csv_dir): print "FAIL: path \"" + csv_dir + "\" does not exist" sys.exit(0) elif item == 'results_server': results_server = yaml_config[item] elif item == 'results_link' : results_link = yaml_config[item] if (results_link != results_server != ''): if not post_file.link_exists(results_server, results_link): print 'WARNING: graph server link does not exist' browser_config = {'preferences' : yaml_config['preferences'], 'extensions' : yaml_config['extensions'], 'browser_path' : yaml_config['browser_path'], 'browser_log' : yaml_config['browser_log'], 'symbols_path' : yaml_config.get('symbols_path', None), 'browser_wait' : yaml_config['browser_wait'], 'process' : yaml_config['process'], 'extra_args' : yaml_config['extra_args'], 'branch' : yaml_config['branch'], 'title' : yaml_config.get('title', ''), 'buildid' : yaml_config['buildid'], 'env' : yaml_config['env'], 'dirs' : yaml_config.get('dirs', {}), 'bundles' : yaml_config.get('bundles', {}), 'init_url' : yaml_config['init_url'], 'child_process' : yaml_config.get('child_process', 'plugin-container'), 'branch_name' : yaml_config.get('branch_name', ''), 'test_name_extension': yaml_config.get('test_name_extension', ''), 'sourcestamp' : yaml_config.get('sourcestamp', 'NULL'), 'repository' : yaml_config.get('repository', 'NULL'), 'host' : yaml_config.get('deviceip', ''), 'port' : yaml_config.get('deviceport', ''), 'webserver' : yaml_config.get('webserver', ''), 'deviceroot' : yaml_config.get('deviceroot', ''), 'remote' : yaml_config.get('remote', False), 'test_timeout' : yaml_config.get('test_timeout', 1200), 'addon_id' : yaml_config.get('addon_id', 'NULL'), 'bcontroller_config' : yaml_config.get('bcontroller_config', 'bcontroller.yml'), 'xperf_path' : yaml_config.get('xperf_path', None)} #normalize paths to work accross platforms dm = None if (browser_config['remote'] == True): import devicemanager if (browser_config['port'] == -1): import devicemanagerADB dm = devicemanagerADB.DeviceManagerADB(browser_config['host'], browser_config['port']) else: import devicemanagerSUT dm = devicemanagerSUT.DeviceManagerSUT(browser_config['host'], browser_config['port']) browser_config['browser_path'] = os.path.normpath(browser_config['browser_path']) for dir in browser_config['dirs']: browser_config['dirs'][dir] = os.path.normpath(browser_config['dirs'][dir]) for bname in browser_config['bundles']: browser_config['bundles'][bname] = os.path.normpath(browser_config['bundles'][bname]) tests = yaml_config['tests'] config_file.close() if (testdate != ''): date = int(time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT'))) else: date = int(time.time()) #TODO get this into own file utils.debug("using testdate: %d" % date) utils.debug("actual date: %d" % int(time.time())) print 'RETURN:s: %s' % title #pull buildid & sourcestamp from browser browser_config = browserInfo(browser_config, devicemanager = dm) if (browser_config['remote'] == True): procName = browser_config['browser_path'].split('/')[-1] if (dm.processExist(procName)): dm.killProcess(procName) utils.startTimer() utils.stamped_msg(title, "Started") for test in tests: testname = test['name'] utils.stamped_msg("Running test " + testname, "Started") try: mytest = TTest(browser_config['remote']) browser_dump, counter_dump, print_format = mytest.runTest(browser_config, test) utils.debug("Received test results: " + " ".join(browser_dump)) results[testname] = [browser_dump, counter_dump, print_format] # If we're doing CSV, write this test immediately (bug 419367) if csv_dir != '': send_to_csv(csv_dir, {testname : results[testname]}) if to_screen or amo: send_to_csv(None, {testname : results[testname]}) except talosError, e: utils.stamped_msg("Failed " + testname, "Stopped") print 'FAIL: Busted: ' + testname print 'FAIL: ' + e.msg.replace('\n','\nRETURN:') utils.stamped_msg("Completed test " + testname, "Stopped")
def send_to_graph(results_server, results_link, title, date, browser_config, results): tbox = title url_format = "http://%s/%s" link_format= "<a href=\"%s\">%s</a>" #value, testname, tbox, timeval, date, branch, buildid, type, data result_format = "%.2f,%s,%s,%d,%d,%s,%s,%s,%s,\n" result_format2 = "%.2f,%s,%s,%d,%d,%s,%s,%s,\n" links = '' for res in results: browser_dump, counter_dump = results[res] utils.debug("Working with test: " + res) utils.debug("Sending results: " + " ".join(browser_dump)) utils.stamped_msg("Transmitting test: " + res, "Started") filename = tempfile.mktemp() tmpf = open(filename, "w") if res in ('ts', 'twinopen'): i = 0 for val in browser_dump: val_list = val.split('|') for v in val_list: tmpf.write(result_format % (float(v), res, tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete", "ms")) i += 1 else: # each line of the string is of the format i;page_name;median;mean;min;max;time vals\n name = '' if ((res == 'tp') or (res == 'tp_js')): name = '_loadtime' for bd in browser_dump: bd.rstrip('\n') page_results = bd.splitlines() i = 0 for mypage in page_results: r = mypage.split(';') #skip this line if it isn't the correct format if len(r) == 1: continue r[1] = r[1].rstrip('/') if r[1].find('/') > -1 : page = r[1].split('/')[1] else: page = r[1] try: val = float(r[2]) except ValueError: print 'WARNING: value error for median in tp' val = 0 tmpf.write(result_format % (val, res + name, tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete", page)) i += 1 tmpf.flush() tmpf.close() links += post_chunk(results_server, results_link, res, filename) os.remove(filename) for cd in counter_dump: for count_type in cd: val_list = cd[count_type] chunks = chunk_list(val_list) chunk_link = '' i = 0 for chunk in chunks: filename = tempfile.mktemp() tmpf = open(filename, "w") for val in chunk: tmpf.write(result_format2 % (float(val), res + "_" + count_type.replace("%", "Percent"), tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete")) i += 1 tmpf.flush() tmpf.close() chunk_link = post_chunk(results_server, results_link, '%s_%s (%d values)' % (res, count_type, len(chunk)), filename) os.remove(filename) links += chunk_link utils.stamped_msg("Transmitting test: " + res, "Stopped") first_results = '' last_results = '' full_results = '\nRETURN:<p style="font-size:smaller;">Details:<br>' lines = links.split('\n') for line in lines: if line == "": continue values = line.split(":") linkName = values[1] if linkName in ('tp_pbytes', 'tp_%cpu'): continue if float(values[2]) > 0: linkName += ": " + str(values[2]) url = url_format % (results_server, values[0]) link = link_format % (url, linkName) first_results = first_results + "\nRETURN:" + link + "<br>" else: url = url_format % (results_server, values[0]) link = link_format % (url, linkName) last_results = last_results + '| ' + link + ' ' full_results = first_results + full_results + last_results + '|</p>' print full_results