def deserialize(f, nm='nm', objdump='objdump', propagateExceptions=False): f = f.name if os.path.getsize(f) == 0: # Empty file - exit early. return None try: data = cPerf.importPerf(f, nm, objdump) # Go through the data and convert counter values to percentages. for f in data['functions'].values(): fc = f['counters'] for l in f['data']: for k,v in l[0].items(): l[0][k] = 100.0 * float(v) / fc[k] for k,v in fc.items(): fc[k] = 100.0 * v / data['counters'][k] return ProfileV1(data) except: if propagateExceptions: raise warning(traceback.format_exc()) return None
def submit_helper(self, parsed_args): """Submit the report to the server. If no server was specified, use a local mock server. """ result = None if parsed_args.submit_url: from lnt.util import ServerUtil for server in parsed_args.submit_url: self.log("submitting result to %r" % (server,)) try: result = ServerUtil.submitFile( server, parsed_args.report_path, parsed_args.commit, parsed_args.verbose) except (urllib2.HTTPError, urllib2.URLError) as e: warning("submitting to {} failed with {}".format(server, e)) else: # Simulate a submission to retrieve the results report. # Construct a temporary database and import the result. self.log("submitting result to dummy instance") import lnt.server.db.v4db import lnt.server.config db = lnt.server.db.v4db.V4DB("sqlite:///:memory:", lnt.server.config.Config.dummyInstance()) result = lnt.util.ImportData.import_and_report( None, None, db, parsed_args.report_path, 'json', True) if result is None: fatal("results were not obtained from submission.") return result
def async_wrapper(job, ts_args, func_args): """Setup test-suite in this subprocess and run something. Because of multipocessing, capture excptions and log messages, and return them. """ global clean_db try: start_time = time.time() if not clean_db: lnt.server.db.v4db.V4DB.close_all_engines() clean_db = True note("Running async wrapper: {} ".format(job.__name__)+ str(os.getpid())) _v4db = current_app.old_config.get_database(ts_args['db']) # with contextlib.closing(_v4db) as db: ts = _v4db.testsuite[ts_args['tsname']] nothing = job(ts, **func_args) assert nothing is None end_time = time.time() delta = end_time-start_time msg = "Finished: {name} in {time:.2f}s ".format(name=job.__name__, time=delta) if delta < 100: note(msg) else: warning(msg) except Exception: # Put all exception text into an exception and raise that for our # parent process. error("Subprocess failed with:" + "".join(traceback.format_exception(*sys.exc_info()))) sys.exit(1) sys.exit(0)
def load_rules(): """ Load available rules scripts from a directory. Rules are organized as: <current dir>/rules/ <current dir>/rules/rule_.*.py ... """ rule_script_rex = re.compile(r'^rule_(.*)\.py$') rule_scripts = {} rules_path = os.path.join(os.path.dirname(__file__), 'rules') for item in os.listdir(rules_path): # Ignore certain known non-scripts. if item in ('README.txt', '__init__.py') or item.endswith('.pyc'): continue # Ignore non-matching files. m = rule_script_rex.match(item) if m is None: warning("ignoring item {} in rule directory: {}".format( item, rules_path)) continue name = m.groups()[0] # Allow rules to be disabled by name if name.endswith("disabled"): continue rule_scripts[name] = os.path.join(rules_path, item) return rule_scripts
def get_source_version(path): """get_source_version(path) -> str or None Given the path to a revision controlled source tree, return a revision number, hash, etc. which identifies the source version. """ if os.path.exists(os.path.join(path, ".svn")): return commands.capture(['/bin/sh', '-c', 'cd "%s" && svnversion' % path]).strip() elif os.path.exists(os.path.join(path, ".git", "svn")): # git-svn is pitifully slow, extract the revision manually. res = commands.capture(['/bin/sh', '-c', ('cd "%s" && ' 'git log -1') % path] ).strip() last_line = res.split("\n")[-1] m = _git_svn_id_re.match(last_line) if not m: commands.warning("unable to understand git svn log: %r" % res) return return m.group(1) elif os.path.exists(os.path.join(path, ".git")): return commands.capture(['/bin/sh', '-c', ('cd "%s" && ' 'git log -1 --pretty=format:%%H') % path] ).strip()
def get_source_version(path): """get_source_version(path) -> str or None Given the path to a revision controlled source tree, return a revision number, hash, etc. which identifies the source version. """ if os.path.exists(os.path.join(path, ".svn")): return commands.capture( ['/bin/sh', '-c', 'cd "%s" && svnversion' % path]).strip() elif os.path.exists(os.path.join(path, ".git", "svn")): # git-svn is pitifully slow, extract the revision manually. res = commands.capture( ['/bin/sh', '-c', ('cd "%s" && ' 'git log -1') % path]).strip() last_line = res.split("\n")[-1] m = _git_svn_id_re.match(last_line) if not m: commands.warning("unable to understand git svn log: %r" % res) return return m.group(1) elif os.path.exists(os.path.join(path, ".git")): return commands.capture([ '/bin/sh', '-c', ('cd "%s" && ' 'git log -1 --pretty=format:%%H') % path ]).strip()
def deserialize(f, nm='nm', objdump='objdump', propagateExceptions=False): f = f.name if os.path.getsize(f) == 0: # Empty file - exit early. return None try: data = cPerf.importPerf(f, nm, objdump) # Go through the data and convert counter values to percentages. for f in data['functions'].values(): fc = f['counters'] for l in f['data']: for k, v in l[0].items(): l[0][k] = 100.0 * float(v) / fc[k] for k, v in fc.items(): fc[k] = 100.0 * v / data['counters'][k] return ProfileV1(data) except: if propagateExceptions: raise warning(traceback.format_exc()) return None
def start_browser(url, debug=False): def url_is_up(url): try: o = urllib.urlopen(url) except IOError: return False o.close() return True # Wait for server to start... if debug: note('waiting for server to start...') for i in range(10000): if url_is_up(url): break if debug: sys.stderr.write('.') sys.stderr.flush() time.sleep(.01) else: warning('unable to detect that server started') if debug: note('opening webbrowser...') webbrowser.open(url)
def action_runtest(name, args): """run a builtin test application""" # Runtest accepting options is deprecated, but lets not break the # world, so collect them anyways and pass them on. parser = OptionParser("%s test-name [options]" % name) parser.disable_interspersed_args() parser.add_option("", "--submit", dest="submit", type=str, default=None) parser.add_option("", "--commit", dest="commit", type=str, default=None) parser.add_option("", "--output", dest="output", type=str, default=None) parser.add_option("-v", "--verbose", dest="verbose", action="store_true") (deprecated_opts, args) = parser.parse_args(args) if len(args) < 1: parser.error("incorrect number of argments") test_name, args = args[0], args[1:] # Rebuild the deprecated arguments. for key, val in vars(deprecated_opts).iteritems(): if val is not None: if isinstance(val, str): args.insert(0, val) args.insert(0, "--" + key) warning("--{} should be passed directly to the" " test suite.".format(key)) import lnt.tests try: test_instance = lnt.tests.get_test_instance(test_name) except KeyError: parser.error('invalid test name %r' % test_name) test_instance.run_test('%s %s' % (name, test_name), args)
def action_submit(name, args): """submit a test report to the server""" parser = OptionParser("%s [options] <url> <file>+" % name) parser.add_option("", "--commit", dest="commit", type=int, help=("whether the result should be committed " "[%default]"), default=True) parser.add_option("-v", "--verbose", dest="verbose", help="show verbose test results", action="store_true", default=False) (opts, args) = parser.parse_args(args) if len(args) < 2: parser.error("incorrect number of argments") if not opts.commit: warning("submit called with --commit=0, your results will not be saved" " at the server.") from lnt.util import ServerUtil files = ServerUtil.submitFiles(args[0], args[1:], opts.commit, opts.verbose) if opts.verbose: for f in files: lnt.util.ImportData.print_report_result(f, sys.stdout, sys.stderr, True)
def async_wrapper(job, ts_args, func_args): """Setup test-suite in this subprocess and run something. Because of multipocessing, capture excptions and log messages, and return them. """ global clean_db try: start_time = time.time() if not clean_db: lnt.server.db.v4db.V4DB.close_all_engines() clean_db = True note("Running async wrapper: {} ".format(job.__name__)+ str(os.getpid())) _v4db = current_app.old_config.get_database(ts_args['db']) #with contextlib.closing(_v4db) as db: ts = _v4db.testsuite[ts_args['tsname']] nothing = job(ts, **func_args) assert nothing is None end_time = time.time() delta = end_time-start_time msg = "Finished: {name} in {time:.2f}s ".format(name=job.__name__, time=delta) if delta < 100: note(msg) else: warning(msg) except: # Put all exception text into an exception and raise that for our # parent process. error("Subprocess failed with:" + "".join(traceback.format_exception(*sys.exc_info()))) sys.exit(1) sys.exit(0)
def submit_helper(self, parsed_args): """Submit the report to the server. If no server was specified, use a local mock server. """ result = None if parsed_args.submit_url: from lnt.util import ServerUtil for server in parsed_args.submit_url: self.log("submitting result to %r" % (server, )) try: result = ServerUtil.submitFile(server, parsed_args.report_path, parsed_args.commit, parsed_args.verbose) except (urllib2.HTTPError, urllib2.URLError) as e: warning("submitting to {} failed with {}".format( server, e)) else: # Simulate a submission to retrieve the results report. # Construct a temporary database and import the result. self.log("submitting result to dummy instance") import lnt.server.db.v4db import lnt.server.config db = lnt.server.db.v4db.V4DB( "sqlite:///:memory:", lnt.server.config.Config.dummyInstance()) result = lnt.util.ImportData.import_and_report( None, None, db, parsed_args.report_path, 'json', True) if result is None: fatal("results were not obtained from submission.") return result
def close(self): t = self.elapsed_time() if t > 10: warning("Request {} took {}s".format(self.url, t)) db = getattr(self, 'db', None) if db is not None: db.close() return super(Request, self).close()
def _importProfile(name_filename): name, filename = name_filename if not os.path.exists(filename): warning('Profile %s does not exist' % filename) return None pf = lnt.testing.profile.profile.Profile.fromFile(filename) if not pf: return None pf.upgrade() profilefile = pf.render() return lnt.testing.TestSamples(name + '.profile', [profilefile], {}, str)
def populate_blacklist(): global ignored ignored = [] try: path = current_app.old_config.blacklist except RuntimeError: path = os.path.join(os.path.dirname(sys.argv[0]), "blacklist") if path and os.path.isfile(path): note("Loading blacklist file: {}".format(path)) with open(path, 'r') as f: for l in f.readlines(): ignored.append(re.compile(l.strip())) else: warning("Ignoring blacklist file: {}".format(path))
def _run_test(self): for iteration in xrange(self.iterations): for output_file in self.output_files: try: os.remove(output_file) except (IOError, OSError): pass try: subprocess.check_call(self.command, cwd=os.getcwd(), shell=True) except subprocess.CalledProcessError: warning("failed to run command: '{}'".format(self.command)) else: self.output.extend([xmltodict.parse(open(output_file, 'r')) for output_file in self.output_files])
def _generate_run_info(self, tag, result_type, run_order, parent_commit): env_vars = { 'Build Number': 'BUILD_NUMBER', 'Owner': 'GERRIT_CHANGE_OWNER_NAME', 'Gerrit URL': 'GERRIT_CHANGE_URL', 'Jenkins URL': 'BUILD_URL' } run_info = { key: os.getenv(env_var) for key, env_var in env_vars.iteritems() if os.getenv(env_var) } try: commit_message = os.getenv('GERRIT_CHANGE_COMMIT_MESSAGE') if commit_message: commit_message = base64.b64decode(commit_message) except Exception: warning('Unable to decode commit message "{}", skipping'.format( commit_message)) else: run_info['Commit Message'] = commit_message git_sha = os.getenv('GERRIT_PATCHSET_REVISION') if not git_sha: fatal("unable to determine git SHA for result, exiting.") if run_order: run_info['run_order'] = str(run_order) else: note("run order not provided, will use server-side auto-generated " "run order") run_info.update({ 'git_sha': git_sha, 't': str(calendar.timegm(time.gmtime())), 'tag': tag }) if result_type == 'cv': if not parent_commit: parent_commit = self._get_parent_commit() run_info.update({'parent_commit': parent_commit}) return run_info
def action_runtest(name, args): """run a builtin test application""" # Runtest accepting options is deprecated, but lets not break the # world, so collect them anyways and pass them on. parser = OptionParser("%s test-name [options]" % name) parser.disable_interspersed_args() parser.add_option("", "--submit", dest="submit", type=str, default=None) parser.add_option("", "--commit", dest="commit", type=str, default=None) parser.add_option("", "--output", dest="output", type=str, default=None) parser.add_option("-v", "--verbose", dest="verbose", action="store_true") (deprecated_opts, args) = parser.parse_args(args) if len(args) < 1: parser.error("incorrect number of argments") test_name, args = args[0], args[1:] # Rebuild the deprecated arguments. for key, val in vars(deprecated_opts).iteritems(): if val is not None: if isinstance(val, str): args.insert(0, val) args.insert(0, "--" + key) warning("--{} should be passed directly to the" " test suite.".format(key)) logger = logging.getLogger(LOGGER_NAME) logger.setLevel(logging.INFO) handler = logging.StreamHandler() handler.setLevel(logging.INFO) handler.setFormatter(logging.Formatter( '%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')) logger.addHandler(handler) import lnt.tests try: test_instance = lnt.tests.get_test_instance(test_name) except KeyError: parser.error('invalid test name %r' % test_name) server_results = test_instance.run_test('%s %s' % (name, test_name), args) if server_results.get('result_url'): print "Results available at:", server_results['result_url'] else: print "Results available at: no URL available"
def action_runtest(name, args): """run a builtin test application""" # Runtest accepting options is deprecated, but lets not break the # world, so collect them anyways and pass them on. parser = OptionParser("%s test-name [options]" % name) parser.disable_interspersed_args() parser.add_option("", "--submit", dest="submit", type=str, default=None) parser.add_option("", "--commit", dest="commit", type=str, default=None) parser.add_option("", "--output", dest="output", type=str, default=None) parser.add_option("-v", "--verbose", dest="verbose", action="store_true") (deprecated_opts, args) = parser.parse_args(args) if len(args) < 1: parser.error("incorrect number of argments") test_name, args = args[0], args[1:] # Rebuild the deprecated arguments. for key, val in vars(deprecated_opts).iteritems(): if val is not None: if isinstance(val, str): args.insert(0, val) args.insert(0, "--" + key) warning("--{} should be passed directly to the" " test suite.".format(key)) logger = logging.getLogger(LOGGER_NAME) logger.setLevel(logging.INFO) handler = logging.StreamHandler() handler.setLevel(logging.INFO) handler.setFormatter( logging.Formatter('%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')) logger.addHandler(handler) import lnt.tests try: test_instance = lnt.tests.get_test_instance(test_name) except KeyError: parser.error('invalid test name %r' % test_name) server_results = test_instance.run_test('%s %s' % (name, test_name), args) if server_results.get('result_url'): print "Results available at:", server_results['result_url'] else: print "Results available at: no URL available"
def _run_test(self): for iteration in xrange(self.iterations): for output_file in self.output_files: try: os.remove(output_file) except (IOError, OSError): pass try: subprocess.check_call(self.command, cwd=os.getcwd(), shell=True) except subprocess.CalledProcessError: warning("failed to run command: '{}'".format(self.command)) else: self.output.extend([ xmltodict.parse(open(output_file, 'r')) for output_file in self.output_files ])
def _generate_run_info(self, tag, result_type, run_order, parent_commit): env_vars = {'Build Number': 'BUILD_NUMBER', 'Owner': 'GERRIT_CHANGE_OWNER_NAME', 'Gerrit URL': 'GERRIT_CHANGE_URL', 'Jenkins URL': 'BUILD_URL'} run_info = {key: os.getenv(env_var) for key, env_var in env_vars.iteritems() if os.getenv(env_var)} try: commit_message = os.getenv('GERRIT_CHANGE_COMMIT_MESSAGE') if commit_message: commit_message = base64.b64decode(commit_message) except Exception: warning('Unable to decode commit message "{}", skipping'.format( commit_message)) else: run_info['Commit Message'] = commit_message git_sha = os.getenv('GERRIT_PATCHSET_REVISION') if not git_sha: fatal("unable to determine git SHA for result, exiting.") if run_order: run_info['run_order'] = str(run_order) else: note("run order not provided, will use server-side auto-generated " "run order") run_info.update({'git_sha': git_sha, 't': str(calendar.timegm(time.gmtime())), 'tag': tag}) if result_type == 'cv': if not parent_commit: parent_commit = self._get_parent_commit() run_info.update({'parent_commit': parent_commit}) return run_info
def load_rules(): """ Load available rules scripts from a directory. Rules are organized as: <current dir>/rules/ <current dir>/rules/rule_.*.py ... """ rule_script_rex = re.compile( r'^rule_(.*)\.py$') rule_scripts = {} rules_path = os.path.join(os.path.dirname(__file__), 'rules') for item in os.listdir(rules_path): # Ignore certain known non-scripts. if item in ('README.txt', '__init__.py') or item.endswith('.pyc'): continue # Ignore non-matching files. m = rule_script_rex.match(item) if m is None: warning("ignoring item {} in rule directory: {}".format(item, rules_path)) continue name = m.groups()[0] # Allow rules to be disabled by name if name.endswith("disabled"): continue rule_scripts[name] = os.path.join(rules_path, item) return rule_scripts
def _parse_lit_output(self, path, data, only_test=False): LIT_METRIC_TO_LNT = { 'compile_time': 'compile', 'exec_time': 'exec', 'score': 'score', 'hash': 'hash' } LIT_METRIC_CONV_FN = { 'compile_time': float, 'exec_time': float, 'score': float, 'hash': str } # We don't use the test info, currently. test_info = {} test_samples = [] # FIXME: Populate with keys not to upload ignore = self.opts.exclude_stat_from_submission if only_test: ignore.append('compile') profiles_to_import = [] for test_data in data['tests']: raw_name = test_data['name'].split(' :: ', 1)[1] name = 'nts.' + raw_name.rsplit('.test', 1)[0] is_pass = self._is_pass_code(test_data['code']) # If --single-result is given, exit based on --single-result-predicate if self.opts.single_result and \ raw_name == self.opts.single_result+'.test': env = {'status': is_pass} if 'metrics' in test_data: for k,v in test_data['metrics'].items(): env[k] = v if k in LIT_METRIC_TO_LNT: env[LIT_METRIC_TO_LNT[k]] = v status = eval(self.opts.single_result_predicate, {}, env) sys.exit(0 if status else 1) if 'metrics' in test_data: for k,v in test_data['metrics'].items(): if k == 'profile': profiles_to_import.append( (name, v) ) continue if k not in LIT_METRIC_TO_LNT or LIT_METRIC_TO_LNT[k] in ignore: continue test_samples.append( lnt.testing.TestSamples(name + '.' + LIT_METRIC_TO_LNT[k], [v], test_info, LIT_METRIC_CONV_FN[k])) if self._test_failed_to_compile(raw_name, path): test_samples.append( lnt.testing.TestSamples(name + '.compile.status', [lnt.testing.FAIL], test_info)) elif not is_pass: test_samples.append( lnt.testing.TestSamples(name + '.exec.status', [self._get_lnt_code(test_data['code'])], test_info)) # Now import the profiles in parallel. if profiles_to_import: note('Importing %d profiles with %d threads...' % (len(profiles_to_import), multiprocessing.cpu_count())) TIMEOUT = 800 try: pool = multiprocessing.Pool() waiter = pool.map_async(_importProfile, profiles_to_import) samples = waiter.get(TIMEOUT) test_samples.extend([sample for sample in samples if sample is not None]) except multiprocessing.TimeoutError: warning('Profiles had not completed importing after %s seconds.' % TIMEOUT) note('Aborting profile import and continuing') if self.opts.single_result: # If we got this far, the result we were looking for didn't exist. raise RuntimeError("Result %s did not exist!" % self.opts.single_result) # FIXME: Add more machine info! run_info = { 'tag': 'nts' } run_info.update(self._get_cc_info()) run_info['run_order'] = run_info['inferred_run_order'] if self.opts.run_order: run_info['run_order'] = self.opts.run_order machine_info = { } machine = lnt.testing.Machine(self.nick, machine_info) run = lnt.testing.Run(self.start_time, timestamp(), info=run_info) report = lnt.testing.Report(machine, run, test_samples) return report
def regenerate_fieldchanges_for_run(ts, run_id): """Regenerate the set of FieldChange objects for the given run. """ # Allow for potentially a few different runs, previous_runs, next_runs # all with the same order_id which we will aggregate together to make # our comparison result. run = ts.getRun(run_id) runs = ts.query(ts.Run). \ filter(ts.Run.order_id == run.order_id). \ filter(ts.Run.machine_id == run.machine_id). \ all() regressions = ts.query(ts.Regression).all()[::-1] previous_runs = ts.get_previous_runs_on_machine(run, FIELD_CHANGE_LOOKBACK) next_runs = ts.get_next_runs_on_machine(run, FIELD_CHANGE_LOOKBACK) # Find our start/end order. if previous_runs != []: start_order = previous_runs[0].order else: start_order = run.order if next_runs != []: end_order = next_runs[-1].order else: end_order = run.order # Load our run data for the creation of the new fieldchanges. runs_to_load = [r.id for r in (runs + previous_runs)] # When the same rev is submitted many times, the database accesses here # can be huge, and it is almost always an error to have the same rev # be used in so many runs. run_size = len(runs_to_load) if run_size > 50: warning("Generating field changes for {} runs." "That will be very slow.".format(run_size)) runinfo = lnt.server.reporting.analysis.RunInfo(ts, runs_to_load) # Only store fieldchanges for "metric" samples like execution time; # not for fields with other data, e.g. hash of a binary for field in list(ts.Sample.get_metric_fields()): for test_id in runinfo.test_ids: f = None result = runinfo.get_comparison_result( runs, previous_runs, test_id, field, ts.Sample.get_hash_of_binary_field()) # Try and find a matching FC and update, else create one. try: f = ts.query(ts.FieldChange) \ .filter(ts.FieldChange.start_order == start_order) \ .filter(ts.FieldChange.end_order == end_order) \ .filter(ts.FieldChange.test_id == test_id) \ .filter(ts.FieldChange.machine == run.machine) \ .filter(ts.FieldChange.field == field) \ .one() except sqlalchemy.orm.exc.NoResultFound: f = None if not result.is_result_performance_change() and f: # With more data, its not a regression. Kill it! note("Removing field change: {}".format(f.id)) delete_fieldchange(ts, f) continue if result.is_result_performance_change() and not f: test = ts.query(ts.Test).filter(ts.Test.id == test_id).one() f = ts.FieldChange(start_order=start_order, end_order=run.order, machine=run.machine, test=test, field=field) ts.add(f) ts.commit() found, new_reg = identify_related_changes(ts, regressions, f) if found: regressions.append(new_reg) note("Found field change: {}".format(run.machine)) # Always update FCs with new values. if f: f.old_value = result.previous f.new_value = result.current f.run = run ts.commit() rules.post_submission_hooks(ts, run_id)
def fromfile(file, path): # I love valgrind, but this is really a horribly lame data format. Oh # well. it = iter(file) # Read the header. description_lines = [] command = None events = None positions = initial_positions = ['line'] for ln in it: # If there is no colon in the line, we have reached the end of the # header. if ':' not in ln: break key, value = ln.split(':', 1) if key == 'desc': description_lines.append(value.strip()) elif key == 'cmd': if command is not None: warning("unexpected multiple 'cmd' keys in %r" % (path, )) command = value.strip() elif key == 'events': if events is not None: warning("unexpected multiple 'events' keys in %r" % (path, )) events = value.split() elif key == 'positions': if positions is not initial_positions: warning("unexpected multiple 'positions' keys in %r" % (path, )) positions = value.split() else: warning("found unknown key %r in %r" % (key, path)) # Validate that required fields were present. if events is None: raise CalltreeParseError("missing required 'events' key in header") # Construct an instance. data = CalltreeData(events, "\n".join(description_lines), command) # Read the file data. num_samples = len(positions) + len(events) current_file = None current_function = None summary_samples = None for ln in it: # Check if this is the closing summary line. if ln.startswith('summary'): key, value = ln.split(':', 1) summary_samples = map(int, value.split()) break # Check if this is an update to the current file or function. if ln.startswith('fl='): current_file = ln[3:-1] elif ln.startswith('fn='): current_function = ln[3:-1] else: # Otherwise, this is a data record. samples = map(int, ln.split()) if len(samples) != num_samples: raise CalltreeParseError( "invalid record line, unexpected sample count") data.records.append((current_file, current_function, samples)) # Validate that there are no more remaining records. for ln in it: raise CalltreeParseError("unexpected line in footer: %r" % (ln, )) # Validate that the summary line was present. if summary_samples is None: raise CalltreeParseError( "missing required 'summary' key in footer") data.summary = summary_samples return data
def regenerate_fieldchanges_for_run(ts, run_id): """Regenerate the set of FieldChange objects for the given run. """ # Allow for potentially a few different runs, previous_runs, next_runs # all with the same order_id which we will aggregate together to make # our comparison result. run = ts.getRun(run_id) runs = ts.query(ts.Run). \ filter(ts.Run.order_id == run.order_id). \ filter(ts.Run.machine_id == run.machine_id). \ all() regressions = ts.query(ts.Regression).all()[::-1] previous_runs = ts.get_previous_runs_on_machine(run, FIELD_CHANGE_LOOKBACK) next_runs = ts.get_next_runs_on_machine(run, FIELD_CHANGE_LOOKBACK) # Find our start/end order. if previous_runs != []: start_order = previous_runs[0].order else: start_order = run.order if next_runs != []: end_order = next_runs[-1].order else: end_order = run.order # Load our run data for the creation of the new fieldchanges. runs_to_load = [r.id for r in (runs + previous_runs)] # When the same rev is submitted many times, the database accesses here # can be huge, and it is almost always an error to have the same rev # be used in so many runs. run_size = len(runs_to_load) if run_size > 50: warning("Generating field changes for {} runs." "That will be very slow.".format(run_size)) runinfo = lnt.server.reporting.analysis.RunInfo(ts, runs_to_load) # Only store fieldchanges for "metric" samples like execution time; # not for fields with other data, e.g. hash of a binary for field in list(ts.Sample.get_metric_fields()): for test_id in runinfo.test_ids: f = None result = runinfo.get_comparison_result( runs, previous_runs, test_id, field, ts.Sample.get_hash_of_binary_field()) # Try and find a matching FC and update, else create one. try: f = ts.query(ts.FieldChange) \ .filter(ts.FieldChange.start_order == start_order) \ .filter(ts.FieldChange.end_order == end_order) \ .filter(ts.FieldChange.test_id == test_id) \ .filter(ts.FieldChange.machine == run.machine) \ .filter(ts.FieldChange.field == field) \ .one() except sqlalchemy.orm.exc.NoResultFound: f = None if not result.is_result_performance_change() and f: # With more data, its not a regression. Kill it! note("Removing field change: {}".format(f.id)) delete_fieldchange(ts, f) continue if result.is_result_performance_change() and not f: test = ts.query(ts.Test).filter(ts.Test.id == test_id).one() f = ts.FieldChange(start_order=start_order, end_order=run.order, machine=run.machine, test=test, field=field) # Check the rules to see if this change matters. if rules.is_useful_change(ts, f): ts.add(f) ts.commit() try: found, new_reg = identify_related_changes( ts, regressions, f) except ObjectDeletedError: # This can happen from time to time. # So, lets retry once. regressions = ts.query(ts.Regression).all()[::-1] found, new_reg = identify_related_changes( ts, regressions, f) if found: regressions.append(new_reg) note("Found field change: {}".format(run.machine)) # Always update FCs with new values. if f: f.old_value = result.previous f.new_value = result.current f.run = run ts.commit() rules.post_submission_hooks(ts, regressions)
def diagnose(self): """Build a triage report that contains information about a test. This is an alternate top level target for running the test-suite. It will produce a triage report for a benchmark instead of running the test-suite normally. The report has stuff in it that will be useful for reproducing and diagnosing a performance change. """ assert self.opts.only_test, "We don't have a benchmark to diagenose." bm_path, short_name = self.opts.only_test assert bm_path, "The benchmark path is empty?" report_name = "{}.report".format(short_name) # Make a place for the report. report_path = os.path.abspath(report_name) # Overwrite the report. if os.path.exists(report_path): shutil.rmtree(report_path) os.mkdir(report_path) path = self._base_path mkdir_p(path) os.chdir(path) # Run with -save-temps cmd = self._configure(path, execute=False) cmd_temps = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-save-temps'] note(' '.join(cmd_temps)) out = subprocess.check_output(cmd_temps) note(out) # Figure out our test's target. make_cmd = [self.opts.make, "VERBOSE=1", 'help'] make_targets = subprocess.check_output(make_cmd) matcher = re.compile(r"^\.\.\.\s{}$".format(short_name), re.MULTILINE | re.IGNORECASE) if not matcher.search(make_targets): assert False, "did not find benchmark, nestsed? Unimplemented." local_path = os.path.join(path, bm_path) make_deps = [self.opts.make, "VERBOSE=1", "timeit-target", "timeit-host", "fpcmp-host"] note(" ".join(make_deps)) p = subprocess.Popen(make_deps, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) std_out, std_err = p.communicate() note(std_out) make_save_temps = [self.opts.make, "VERBOSE=1", short_name] note(" ".join(make_save_temps)) p = subprocess.Popen(make_save_temps, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) std_out, std_err = p.communicate() note(std_out) with open(report_path + "/build.log", 'w') as f: f.write(std_out) # Executable(s) and test file: shutil.copy(os.path.join(local_path, short_name), report_path) shutil.copy(os.path.join(local_path, short_name + ".test"), report_path) # Temp files are in: temp_files = os.path.join(local_path, "CMakeFiles", short_name + ".dir") save_temps_file = ["/*.s", "/*.ii", "/*.i", "/*.bc"] build_files = ["/*.o", "/*.time", "/*.cmake", "/*.make", "/*.includecache", "/*.txt"] self._cp_artifacts(local_path, report_path, save_temps_file) self._cp_artifacts(temp_files, report_path, build_files) # Now lets do -ftime-report. cmd_time_report = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-ftime-report'] note(' '.join(cmd_time_report)) out = subprocess.check_output(cmd_time_report) note(out) make_time_report = [self.opts.make, "VERBOSE=1", short_name] p = subprocess.Popen(make_time_report, stdout=subprocess.PIPE, stderr=subprocess.PIPE) std_out, std_err = p.communicate() with open(report_path + "/time-report.txt", 'w') as f: f.write(std_err) note("Wrote: " + report_path + "/time-report.txt") # Now lets do -llvm -stats. cmd_stats_report = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-mllvm -stats'] note(' '.join(cmd_stats_report)) out = subprocess.check_output(cmd_stats_report) note(out) make_stats_report = [self.opts.make, "VERBOSE=1", short_name] p = subprocess.Popen(make_stats_report, stdout=subprocess.PIPE, stderr=subprocess.PIPE) std_out, std_err = p.communicate() with open(report_path + "/stats-report.txt", 'w') as f: f.write(std_err) note("Wrote: " + report_path + "/stats-report.txt") # Collect Profile: if "Darwin" in platform.platform(): # For testing and power users, lets allow overrides of how sudo # and iprofiler are called. sudo = os.getenv("SUDO_CMD", "sudo") if " " in sudo: sudo = sudo.split(" ") if not sudo: sudo = [] else: sudo = [sudo] iprofiler = os.getenv("IPROFILER_CMD", "iprofiler -timeprofiler -I 40u") cmd_iprofiler = cmd + ['-DTEST_SUITE_RUN_UNDER=' + iprofiler] print ' '.join(cmd_iprofiler) out = subprocess.check_output(cmd_iprofiler) os.chdir(local_path) make_iprofiler_temps = [self.opts.make, "VERBOSE=1", short_name] p = subprocess.Popen(make_iprofiler_temps, stdout=subprocess.PIPE, stderr=subprocess.PIPE) std_out, std_err = p.communicate() warning("Using sudo to collect execution trace.") make_save_temps = sudo + [self.opts.lit, short_name + ".test"] p = subprocess.Popen(make_save_temps, stdout=subprocess.PIPE, stderr=subprocess.PIPE) std_out, std_err = p.communicate() sys.stdout.write(std_out) sys.stderr.write(std_err) warning("Tests may fail because of iprofiler's output.") # The dtps file will be saved as root, make it so # that we can read it. chmod = sudo + ["chown", "-R", getpass.getuser(), short_name + ".dtps"] subprocess.call(chmod) profile = local_path + "/" + short_name + ".dtps" shutil.copytree(profile, report_path + "/" + short_name + ".dtps") note(profile + "-->" + report_path) else: warning("Skipping execution profiling because this is not Darwin.") note("Report produced in: " + report_path) # Run through the rest of LNT, but don't allow this to be submitted # because there is no data. class DontSubmitResults(object): def get(self, url): return report_path def __getitem__(self, key): return report_path return DontSubmitResults()
def fromfile(file, path): # I love valgrind, but this is really a horribly lame data format. Oh # well. it = iter(file) # Read the header. description_lines = [] command = None events = None positions = initial_positions = ['line'] for ln in it: # If there is no colon in the line, we have reached the end of the # header. if ':' not in ln: break key,value = ln.split(':', 1) if key == 'desc': description_lines.append(value.strip()) elif key == 'cmd': if command is not None: warning("unexpected multiple 'cmd' keys in %r" % (path,)) command = value.strip() elif key == 'events': if events is not None: warning("unexpected multiple 'events' keys in %r" % (path,)) events = value.split() elif key == 'positions': if positions is not initial_positions: warning("unexpected multiple 'positions' keys in %r" % ( path,)) positions = value.split() else: warning("found unknown key %r in %r" % (key, path)) # Validate that required fields were present. if events is None: raise CalltreeParseError("missing required 'events' key in header") # Construct an instance. data = CalltreeData(events, "\n".join(description_lines), command) # Read the file data. num_samples = len(positions) + len(events) current_file = None current_function = None summary_samples = None for ln in it: # Check if this is the closing summary line. if ln.startswith('summary'): key,value = ln.split(':', 1) summary_samples = map(int, value.split()) break # Check if this is an update to the current file or function. if ln.startswith('fl='): current_file = ln[3:-1] elif ln.startswith('fn='): current_function = ln[3:-1] else: # Otherwise, this is a data record. samples = map(int, ln.split()) if len(samples) != num_samples: raise CalltreeParseError( "invalid record line, unexpected sample count") data.records.append((current_file, current_function, samples)) # Validate that there are no more remaining records. for ln in it: raise CalltreeParseError("unexpected line in footer: %r" % (ln,)) # Validate that the summary line was present. if summary_samples is None: raise CalltreeParseError("missing required 'summary' key in footer") data.summary = summary_samples return data
def _parse_lit_output(self, path, data, only_test=False): LIT_METRIC_TO_LNT = { 'compile_time': 'compile', 'exec_time': 'exec', 'score': 'score', 'hash': 'hash' } LIT_METRIC_CONV_FN = { 'compile_time': float, 'exec_time': float, 'score': float, 'hash': str } # We don't use the test info, currently. test_info = {} test_samples = [] # FIXME: Populate with keys not to upload ignore = self.opts.exclude_stat_from_submission if only_test: ignore.append('compile') profiles_to_import = [] for test_data in data['tests']: raw_name = test_data['name'].split(' :: ', 1)[1] name = 'nts.' + raw_name.rsplit('.test', 1)[0] is_pass = self._is_pass_code(test_data['code']) # If --single-result is given, exit based on --single-result-predicate if self.opts.single_result and \ raw_name == self.opts.single_result+'.test': env = {'status': is_pass} if 'metrics' in test_data: for k, v in test_data['metrics'].items(): env[k] = v if k in LIT_METRIC_TO_LNT: env[LIT_METRIC_TO_LNT[k]] = v status = eval(self.opts.single_result_predicate, {}, env) sys.exit(0 if status else 1) if 'metrics' in test_data: for k, v in test_data['metrics'].items(): if k == 'profile': profiles_to_import.append((name, v)) continue if k not in LIT_METRIC_TO_LNT or LIT_METRIC_TO_LNT[ k] in ignore: continue test_samples.append( lnt.testing.TestSamples( name + '.' + LIT_METRIC_TO_LNT[k], [v], test_info, LIT_METRIC_CONV_FN[k])) if self._test_failed_to_compile(raw_name, path): test_samples.append( lnt.testing.TestSamples(name + '.compile.status', [lnt.testing.FAIL], test_info)) elif not is_pass: test_samples.append( lnt.testing.TestSamples( name + '.exec.status', [self._get_lnt_code(test_data['code'])], test_info)) # Now import the profiles in parallel. if profiles_to_import: note('Importing %d profiles with %d threads...' % (len(profiles_to_import), multiprocessing.cpu_count())) TIMEOUT = 800 try: pool = multiprocessing.Pool() waiter = pool.map_async(_importProfile, profiles_to_import) samples = waiter.get(TIMEOUT) test_samples.extend( [sample for sample in samples if sample is not None]) except multiprocessing.TimeoutError: warning( 'Profiles had not completed importing after %s seconds.' % TIMEOUT) note('Aborting profile import and continuing') if self.opts.single_result: # If we got this far, the result we were looking for didn't exist. raise RuntimeError("Result %s did not exist!" % self.opts.single_result) # FIXME: Add more machine info! run_info = {'tag': 'nts'} run_info.update(self._get_cc_info()) run_info['run_order'] = run_info['inferred_run_order'] if self.opts.run_order: run_info['run_order'] = self.opts.run_order machine_info = {} machine = lnt.testing.Machine(self.nick, machine_info) run = lnt.testing.Run(self.start_time, timestamp(), info=run_info) report = lnt.testing.Report(machine, run, test_samples) return report
def action_updatedb(name, args): """modify a database""" from optparse import OptionParser, OptionGroup parser = OptionParser("%s [options] <instance> <file>+" % name) parser.add_option("", "--database", dest="database", default="default", help="database to modify [%default]") parser.add_option("", "--testsuite", dest="testsuite", help="testsuite to modify") parser.add_option("", "--commit", dest="commit", type=int, default=False) parser.add_option("", "--show-sql", dest="show_sql", action="store_true", default=False) parser.add_option("", "--delete-machine", dest="delete_machines", action="append", default=[]) parser.add_option("", "--delete-run", dest="delete_runs", action="append", default=[], type=int) (opts, args) = parser.parse_args(args) if len(args) != 1: parser.error("invalid number of arguments") if opts.testsuite is None: parser.error("--testsuite is required") path, = args # Load the instance. instance = lnt.server.instance.Instance.frompath(path) # Get the database and test suite. with contextlib.closing( instance.get_database(opts.database, echo=opts.show_sql)) as db: ts = db.testsuite[opts.testsuite] # Compute a list of all the runs to delete. runs_to_delete = list(opts.delete_runs) if opts.delete_machines: runs_to_delete.extend( id for id, in ts.query(ts.Run.id).\ join(ts.Machine).\ filter(ts.Machine.name.in_(opts.delete_machines))) # Delete all samples associated with those runs. ts.query(ts.Sample).\ filter(ts.Sample.run_id.in_(runs_to_delete)).\ delete(synchronize_session=False) # Delete all those runs. ts.query(ts.Run).\ filter(ts.Run.id.in_(runs_to_delete)).\ delete(synchronize_session=False) # Delete the machines. for name in opts.delete_machines: # Delete all FieldChanges associated with this machine. ids = ts.query(ts.FieldChange.id).\ join(ts.Machine).filter(ts.Machine.name == name).all() for i in ids: ts.query(ts.FieldChange).filter(ts.FieldChange.id == i[0]).\ delete() num_deletes = ts.query(ts.Machine).filter_by(name=name).delete() if num_deletes == 0: warning("unable to find machine named: %r" % name) if opts.commit: db.commit() else: db.rollback()
def action_updatedb(name, args): """modify a database""" from optparse import OptionParser, OptionGroup parser = OptionParser("%s [options] <instance> <file>+"%name) parser.add_option("", "--database", dest="database", default="default", help="database to modify [%default]") parser.add_option("", "--testsuite", dest="testsuite", help="testsuite to modify") parser.add_option("", "--commit", dest="commit", type=int, default=False) parser.add_option("", "--show-sql", dest="show_sql", action="store_true", default=False) parser.add_option("", "--delete-machine", dest="delete_machines", action="append", default=[]) parser.add_option("", "--delete-run", dest="delete_runs", action="append", default=[], type=int) (opts, args) = parser.parse_args(args) if len(args) != 1: parser.error("invalid number of arguments") if opts.testsuite is None: parser.error("--testsuite is required") path, = args # Load the instance. instance = lnt.server.instance.Instance.frompath(path) # Get the database and test suite. with contextlib.closing(instance.get_database(opts.database, echo=opts.show_sql)) as db: ts = db.testsuite[opts.testsuite] # Compute a list of all the runs to delete. runs_to_delete = list(opts.delete_runs) if opts.delete_machines: runs_to_delete.extend( id for id, in ts.query(ts.Run.id).\ join(ts.Machine).\ filter(ts.Machine.name.in_(opts.delete_machines))) # Delete all samples associated with those runs. ts.query(ts.Sample).\ filter(ts.Sample.run_id.in_(runs_to_delete)).\ delete(synchronize_session=False) # Delete all those runs. ts.query(ts.Run).\ filter(ts.Run.id.in_(runs_to_delete)).\ delete(synchronize_session=False) # Delete the machines. for name in opts.delete_machines: # Delete all FieldChanges associated with this machine. ids = ts.query(ts.FieldChange.id).\ join(ts.Machine).filter(ts.Machine.name == name).all() for i in ids: ts.query(ts.FieldChange).filter(ts.FieldChange.id == i[0]).\ delete() num_deletes = ts.query(ts.Machine).filter_by(name=name).delete() if num_deletes == 0: warning("unable to find machine named: %r" % name) if opts.commit: db.commit() else: db.rollback()