def __enter__(self):
self._saved = os.dup(sys.stdout.fileno()), os.dup(sys.stderr.fileno())
self.tee = subprocess.Popen(["tee", self.filepath], stdin=subprocess.PIPE)
os.dup2(self.tee.stdin.fileno(), sys.stdout.fileno())
os.dup2(self.tee.stdin.fileno(), sys.stderr.fileno())
if self.time:
print datetime()
self.start_time = time.time()
def report_ticket(self, ticket, status, log, plugins=(), dry_run=False, pending_status=None):
report = {
'status': status,
'patches': ticket['patches'],
'deps': ticket['depends_on'],
'spkgs': ticket['spkgs'],
'base': self.base,
'user': self.config['user'],
'machine': self.config['machine'],
'time': datetime(),
'plugins': plugins,
'patchbot_version': patchbot_version.get_version(),
}
if pending_status:
report['pending_status'] = pending_status
try:
report['base'] = ticket_base = sorted([
describe_branch('patchbot/base', tag_only=True),
describe_branch('patchbot/ticket_upstream', tag_only=True)], compare_version)[-1]
report['git_base'] = self.git_commit('patchbot/base')
report['git_base_human'] = describe_branch('patchbot/base')
if ticket['id'] != 0:
report['git_branch'] = ticket.get('git_branch', None)
report['git_log'] = subprocess.check_output(['git', 'log', '--oneline', '%s..patchbot/ticket_upstream' % ticket_base]).strip().split('\n')
# If apply failed, we don't want to be stuck in an infinite loop.
report['git_commit'] = self.git_commit('patchbot/ticket_upstream')
report['git_commit_human'] = describe_branch('patchbot/ticket_upstream')
report['git_merge'] = self.git_commit('patchbot/ticket_merged')
report['git_merge_human'] = describe_branch('patchbot/ticket_merged')
else:
report['git_branch'] = self.config['base_branch']
report['git_log'] = []
report['git_commit'] = report['git_merge'] = report['git_base']
except Exception:
traceback.print_exc()
if status != 'Pending':
history = open("%s/history.txt" % self.log_dir, "a")
history.write("%s %s %s%s\n" % (
datetime(),
ticket['id'],
status,
" dry_run" if dry_run else ""))
history.close()
print "REPORT"
import pprint
pprint.pprint(report)
print ticket['id'], status
fields = {'report': json.dumps(report)}
if os.path.exists(log):
files = [('log', 'log', bz2.compress(open(log).read()))]
else:
files = []
if not dry_run or status == 'Pending':
print post_multipart("%s/report/%s" % (self.server, ticket['id']), fields, files)
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None:
traceback.print_exc()
if self.time:
print datetime()
print int(time.time() - self.start_time), "seconds"
self.tee.stdin.close()
time.sleep(1)
os.dup2(self._saved[0], sys.stdout.fileno())
os.dup2(self._saved[1], sys.stderr.fileno())
time.sleep(1)
try:
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(self.timeout)
self.tee.wait()
signal.alarm(0)
except TimeOut:
traceback.print_exc()
raise
return False
def report_ticket(server, ticket, status, base, machine, user, log, plugins=[]):
print ticket['id'], status
report = {
'status': status,
'patches': ticket['patches'],
'deps': ticket['depends_on'],
'spkgs': ticket['spkgs'],
'base': base,
'user': user,
'machine': machine,
'time': datetime(),
'plugins': plugins,
}
fields = {'report': json.dumps(report)}
if status != 'Pending':
files = [('log', 'log', bz2.compress(open(log).read()))]
else:
files = []
print post_multipart("%s/report/%s" % (server, ticket['id']), fields, files)
def test_a_ticket(self, ticket=None):
self.reload_config()
if ticket is None:
ticket = self.get_ticket()
else:
ticket = None, scrape(int(ticket))
if not ticket:
print "No more tickets."
time.sleep(self.config['idle'])
return
rating, ticket = ticket
print "\n" * 2
print "=" * 30, ticket['id'], "=" * 30
print ticket['title']
print "score", rating
print "\n" * 2
self.log_dir = self.sage_root + "/logs/patchbot"
if not os.path.exists(self.log_dir):
os.makedirs(self.log_dir)
log = '%s/%s-log.txt' % (self.log_dir, ticket['id'])
history = open("%s/history.txt" % self.log_dir, "a")
history.write("%s %s\n" % (datetime(), ticket['id']))
history.close()
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=log)
plugins_results = []
try:
t = Timer()
with Tee(log, time=True, timeout=self.config['timeout'], timer=t):
print "Sage Patchbot", patchbot_version.get_version()
if ticket['spkgs']:
state = 'spkg'
print "\n".join(ticket['spkgs'])
print
for spkg in ticket['spkgs']:
print
print '+' * 10, spkg, '+' * 10
print
try:
self.check_spkg(spkg)
except Exception:
traceback.print_exc()
t.finish(spkg)
if not ticket['spkgs']:
state = 'started'
os.environ['MAKE'] = "make -j%s" % self.config['parallelism']
os.environ['SAGE_ROOT'] = self.sage_root
os.environ['GIT_AUTHOR_NAME'] = os.environ['GIT_COMMITTER_NAME'] = 'patchbot'
os.environ['GIT_AUTHOR_EMAIL'] = os.environ['GIT_COMMITTER_EMAIL'] = 'patchbot@localhost'
os.environ['GIT_AUTHOR_DATE'] = os.environ['GIT_COMMITTER_DATE'] = '1970-01-01T00:00:00'
pull_from_trac(self.sage_root, ticket['id'], force=True, use_ccache=self.config['use_ccache'])
t.finish("Apply")
state = 'applied'
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=log, pending_status=state)
do_or_die("$MAKE")
t.finish("Build")
state = 'built'
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=log, pending_status=state)
# TODO: Exclude dependencies.
patch_dir = tempfile.mkdtemp()
if ticket['id'] != 0:
do_or_die("git format-patch -o '%s' patchbot/base..patchbot/ticket_merged" % patch_dir)
kwds = {
"original_dir": self.sage_root,
"patched_dir": os.getcwd(),
"patches": [os.path.join(patch_dir, p) for p in os.listdir(patch_dir)],
"sage_binary": os.path.join(os.getcwd(), 'sage'),
"dry_run": self.dry_run,
}
for name, plugin in self.config['plugins']:
try:
if ticket['id'] != 0 and os.path.exists(os.path.join(self.log_dir, '0', name)):
baseline = pickle.load(open(os.path.join(self.log_dir, '0', name)))
else:
baseline = None
print plugin_boundary(name)
do_or_die("git checkout patchbot/ticket_merged")
res = plugin(ticket, is_git=True, baseline=baseline, **kwds)
passed = True
except Exception:
traceback.print_exc()
passed = False
res = None
finally:
if isinstance(res, PluginResult):
if res.baseline is not None:
plugin_dir = os.path.join(self.log_dir, str(ticket['id']))
if not os.path.exists(plugin_dir):
os.mkdir(plugin_dir)
pickle.dump(res.baseline, open(os.path.join(plugin_dir, name), 'w'))
passed = res.status == PluginResult.Passed
print name, res.status
plugins_results.append((name, passed, res.data))
else:
plugins_results.append((name, passed, None))
t.finish(name)
print plugin_boundary(name, end=True)
plugins_passed = all(passed for (name, passed, data) in plugins_results)
self.report_ticket(ticket, status='Pending', log=log, pending_status='plugins_passed' if plugins_passed else 'plugins_failed')
if self.plugin_only:
state = 'plugins' if plugins_passed else 'plugins_failed'
else:
if self.dry_run:
test_target = "$SAGE_ROOT/src/sage/misc/a*.py"
# TODO: Remove
test_target = "$SAGE_ROOT/src/sage/doctest/*.py"
else:
test_target = "--all --long"
if self.config['parallelism'] > 1:
test_cmd = "-tp %s" % self.config['parallelism']
else:
test_cmd = "-t"
do_or_die("$SAGE_ROOT/sage %s %s" % (test_cmd, test_target))
t.finish("Tests")
state = 'tested'
if not plugins_passed:
state = 'tests_passed_plugins_failed'
except (urllib2.HTTPError, socket.error):
# Don't report failure because the network/trac died...
print
t.print_all()
traceback.print_exc()
state = 'network_error'
except Exception:
traceback.print_exc()
for _ in range(5):
try:
print "Reporting", ticket['id'], status[state]
self.report_ticket(ticket, status=status[state], log=log, plugins=plugins_results, dry_run=self.dry_run)
print "Done reporting", ticket['id']
break
except IOError:
traceback.print_exc()
time.sleep(self.config['idle'])
else:
print "Error reporting", ticket['id']
maybe_temp_root = os.environ['SAGE_ROOT']
if maybe_temp_root.endswith("-sage-git-temp-%s" % ticket['id']):
shutil.rmtree(maybe_temp_root)
return status[state]