def test_a_ticket(self, ticket=None): self.reload_config() if ticket is None: ticket = self.get_ticket() else: ticket = None, scrape(int(ticket)) if not ticket: print "No more tickets." time.sleep(self.config['idle']) return rating, ticket = ticket print "\n" * 2 print "=" * 30, ticket['id'], "=" * 30 print ticket['title'] print "score", rating print "\n" * 2 self.log_dir = self.sage_root + "/logs/patchbot" if not os.path.exists(self.log_dir): os.makedirs(self.log_dir) log = '%s/%s-log.txt' % (self.log_dir, ticket['id']) history = open("%s/history.txt" % self.log_dir, "a") history.write("%s %s\n" % (datetime(), ticket['id'])) history.close() if not self.plugin_only: self.report_ticket(ticket, status='Pending', log=log) plugins_results = [] try: t = Timer() with Tee(log, time=True, timeout=self.config['timeout'], timer=t): print "Sage Patchbot", patchbot_version.get_version() if ticket['spkgs']: state = 'spkg' print "\n".join(ticket['spkgs']) print for spkg in ticket['spkgs']: print print '+' * 10, spkg, '+' * 10 print try: self.check_spkg(spkg) except Exception: traceback.print_exc() t.finish(spkg) if not ticket['spkgs']: state = 'started' os.environ['MAKE'] = "make -j%s" % self.config['parallelism'] os.environ['SAGE_ROOT'] = self.sage_root os.environ['GIT_AUTHOR_NAME'] = os.environ['GIT_COMMITTER_NAME'] = 'patchbot' os.environ['GIT_AUTHOR_EMAIL'] = os.environ['GIT_COMMITTER_EMAIL'] = 'patchbot@localhost' os.environ['GIT_AUTHOR_DATE'] = os.environ['GIT_COMMITTER_DATE'] = '1970-01-01T00:00:00' pull_from_trac(self.sage_root, ticket['id'], force=True, use_ccache=self.config['use_ccache']) t.finish("Apply") state = 'applied' if not self.plugin_only: self.report_ticket(ticket, status='Pending', log=log, pending_status=state) do_or_die("$MAKE") t.finish("Build") state = 'built' if not self.plugin_only: self.report_ticket(ticket, status='Pending', log=log, pending_status=state) # TODO: Exclude dependencies. patch_dir = tempfile.mkdtemp() if ticket['id'] != 0: do_or_die("git format-patch -o '%s' patchbot/base..patchbot/ticket_merged" % patch_dir) kwds = { "original_dir": self.sage_root, "patched_dir": os.getcwd(), "patches": [os.path.join(patch_dir, p) for p in os.listdir(patch_dir)], "sage_binary": os.path.join(os.getcwd(), 'sage'), "dry_run": self.dry_run, } for name, plugin in self.config['plugins']: try: if ticket['id'] != 0 and os.path.exists(os.path.join(self.log_dir, '0', name)): baseline = pickle.load(open(os.path.join(self.log_dir, '0', name))) else: baseline = None print plugin_boundary(name) do_or_die("git checkout patchbot/ticket_merged") res = plugin(ticket, is_git=True, baseline=baseline, **kwds) passed = True except Exception: traceback.print_exc() passed = False res = None finally: if isinstance(res, PluginResult): if res.baseline is not None: plugin_dir = os.path.join(self.log_dir, str(ticket['id'])) if not os.path.exists(plugin_dir): os.mkdir(plugin_dir) pickle.dump(res.baseline, open(os.path.join(plugin_dir, name), 'w')) passed = res.status == PluginResult.Passed print name, res.status plugins_results.append((name, passed, res.data)) else: plugins_results.append((name, passed, None)) t.finish(name) print plugin_boundary(name, end=True) plugins_passed = all(passed for (name, passed, data) in plugins_results) self.report_ticket(ticket, status='Pending', log=log, pending_status='plugins_passed' if plugins_passed else 'plugins_failed') if self.plugin_only: state = 'plugins' if plugins_passed else 'plugins_failed' else: if self.dry_run: test_target = "$SAGE_ROOT/src/sage/misc/a*.py" # TODO: Remove test_target = "$SAGE_ROOT/src/sage/doctest/*.py" else: test_target = "--all --long" if self.config['parallelism'] > 1: test_cmd = "-tp %s" % self.config['parallelism'] else: test_cmd = "-t" do_or_die("$SAGE_ROOT/sage %s %s" % (test_cmd, test_target)) t.finish("Tests") state = 'tested' if not plugins_passed: state = 'tests_passed_plugins_failed' except (urllib2.HTTPError, socket.error): # Don't report failure because the network/trac died... print t.print_all() traceback.print_exc() state = 'network_error' except Exception: traceback.print_exc() for _ in range(5): try: print "Reporting", ticket['id'], status[state] self.report_ticket(ticket, status=status[state], log=log, plugins=plugins_results, dry_run=self.dry_run) print "Done reporting", ticket['id'] break except IOError: traceback.print_exc() time.sleep(self.config['idle']) else: print "Error reporting", ticket['id'] maybe_temp_root = os.environ['SAGE_ROOT'] if maybe_temp_root.endswith("-sage-git-temp-%s" % ticket['id']): shutil.rmtree(maybe_temp_root) return status[state]
def test_a_ticket(sage_root, server, ticket=None, nodocs=False): base = get_base(sage_root) if ticket is None: ticket = get_ticket(base=base, server=server, **conf) else: ticket = None, scrape(int(ticket)) if not ticket: print "No more tickets." if random.random() < 0.01: cleanup(sage_root, server) time.sleep(conf['idle']) return rating, ticket = ticket print "\n" * 2 print "=" * 30, ticket['id'], "=" * 30 print ticket['title'] print "score", rating print "\n" * 2 log_dir = sage_root + "/logs" if not os.path.exists(log_dir): os.mkdir(log_dir) log = '%s/%s-log.txt' % (log_dir, ticket['id']) report_ticket(server, ticket, status='Pending', base=base, machine=conf['machine'], user=conf['user'], log=None) plugins_results = [] try: with Tee(log, time=True, timeout=conf['timeout']): t = Timer() start_time = time.time() state = 'started' os.environ['MAKE'] = "make -j%s" % conf['parallelism'] os.environ['SAGE_ROOT'] = sage_root # TODO: Ensure that sage-main is pristine. pull_from_trac(sage_root, ticket['id'], force=True) t.finish("Apply") state = 'applied' do_or_die('$SAGE_ROOT/sage -b %s' % ticket['id']) t.finish("Build") state = 'built' working_dir = "%s/devel/sage-%s" % (sage_root, ticket['id']) # Only the ones on this ticket. patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):] kwds = { "original_dir": "%s/devel/sage-0" % sage_root, "patched_dir": working_dir, "patches": ["%s/devel/sage-%s/.hg/patches/%s" % (sage_root, ticket['id'], p) for p in patches if p], } for name, plugin in conf['plugins']: try: print plugin_boundary(name) plugin(ticket, **kwds) passed = True except Exception: traceback.print_exc() passed = False finally: t.finish(name) print plugin_boundary(name, end=True) plugins_results.append((name, passed)) test_dirs = ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs] if conf['parallelism'] > 1: test_cmd = "-tp %s" % conf['parallelism'] else: test_cmd = "-t" do_or_die("$SAGE_ROOT/sage %s -sagenb %s" % (test_cmd, ' '.join(test_dirs))) #do_or_die("$SAGE_ROOT/sage -t $SAGE_ROOT/devel/sage-%s/sage/rings/integer.pyx" % ticket['id']) #do_or_die('sage -testall') t.finish("Tests") state = 'tested' if not all(passed for name, passed in plugins_results): state = 'failed_plugin' print t.print_all() except Exception: traceback.print_exc() for _ in range(5): try: print "Reporting", ticket['id'], status[state] report_ticket(server, ticket, status=status[state], base=base, machine=conf['machine'], user=conf['user'], log=log, plugins=plugins_results) print "Done reporting", ticket['id'] break except urllib2.HTTPError: traceback.print_exc() time.sleep(conf['idle']) else: print "Error reporting", ticket['id'] return status[state]
def test_a_ticket(self, ticket=None): self.reload_config() if ticket is None: ticket = self.get_ticket() else: ticket = None, scrape(int(ticket)) if not ticket: print "No more tickets." time.sleep(conf['idle']) return rating, ticket = ticket print "\n" * 2 print "=" * 30, ticket['id'], "=" * 30 print ticket['title'] print "score", rating print "\n" * 2 log_dir = self.sage_root + "/logs" if not os.path.exists(log_dir): os.mkdir(log_dir) log = '%s/%s-log.txt' % (log_dir, ticket['id']) if not self.plugin_only: self.report_ticket(ticket, status='Pending', log=None) plugins_results = [] try: with Tee(log, time=True, timeout=self.config['timeout']): t = Timer() start_time = time.time() if ticket['spkgs']: state = 'spkg' print "\n".join(ticket['spkgs']) print for spkg in ticket['spkgs']: print print '+' * 10, spkg, '+' * 10 print try: self.check_spkg(spkg) except Exception: traceback.print_exc() t.finish(spkg) else: state = 'started' os.environ['MAKE'] = "make -j%s" % self.config['parallelism'] os.environ['SAGE_ROOT'] = self.sage_root # TODO: Ensure that sage-main is pristine. pull_from_trac(self.sage_root, ticket['id'], force=True) t.finish("Apply") state = 'applied' do_or_die('$SAGE_ROOT/sage -b %s' % ticket['id']) t.finish("Build") state = 'built' working_dir = "%s/devel/sage-%s" % (self.sage_root, ticket['id']) # Only the ones on this ticket. patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):] kwds = { "original_dir": "%s/devel/sage-0" % self.sage_root, "patched_dir": working_dir, "patches": ["%s/devel/sage-%s/.hg/patches/%s" % (self.sage_root, ticket['id'], p) for p in patches if p], "sage_binary": os.path.join(self.sage_root, 'sage') } for name, plugin in self.config['plugins']: try: if ticket['id'] != 0 and os.path.exists(os.path.join(log_dir, '0', name)): baseline = pickle.load(open(os.path.join(log_dir, '0', name))) else: baseline = None print plugin_boundary(name) res = plugin(ticket, baseline=baseline, **kwds) passed = True except Exception: traceback.print_exc() passed = False res = None finally: if isinstance(res, PluginResult): if res.baseline is not None: plugin_dir = os.path.join(log_dir, str(ticket['id'])) if not os.path.exists(plugin_dir): os.mkdir(plugin_dir) pickle.dump(res.baseline, open(os.path.join(plugin_dir, name), 'w')) passed = res.status == PluginResult.Passed print name, res.status plugins_results.append((name, passed, res.data)) else: plugins_results.append((name, passed, None)) t.finish(name) print plugin_boundary(name, end=True) plugins_passed = all(passed for (name, passed, data) in plugins_results) if self.plugin_only: state = 'plugins' if plugins_passed else 'plugins_failed' else: if self.dry_run: test_dirs = ["$SAGE_ROOT/devel/sage-%s/sage/misc/a*.py" % (ticket['id'])] else: test_dirs = ["--sagenb"] + ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs] if conf['parallelism'] > 1: test_cmd = "-tp %s" % conf['parallelism'] else: test_cmd = "-t" do_or_die("$SAGE_ROOT/sage %s %s" % (test_cmd, ' '.join(test_dirs))) t.finish("Tests") state = 'tested' if not plugins_passed: state = 'tests_passed_plugins_failed' print t.print_all() except urllib2.HTTPError: # Don't report failure because the network/trac died... traceback.print_exc() return 'Pending' except Exception: traceback.print_exc() for _ in range(5): try: print "Reporting", ticket['id'], status[state] if not self.dry_run: self.report_ticket(ticket, status=status[state], log=log, plugins=plugins_results) print "Done reporting", ticket['id'] break except urllib2.HTTPError: traceback.print_exc() time.sleep(conf['idle']) else: print "Error reporting", ticket['id'] if not conf['keep_open_branches'] and str(ticket['id']) != '0' and not ticket['spkgs']: shutil.rmtree(os.path.join(self.sage_root, "devel", "sage-%s" %ticket['id'])) return status[state]