def pull_from_trac(sage_root, ticket_id, branch=None, force=None,
use_ccache=False,
safe_only=False):
"""
Create four branches from base and ticket.
If ticket deemed unsafe then clone git repo to temp directory. ?!
Additionally, if ``use_ccache`` then install ccache. Set some global
and environment variables.
There are four branches at play here:
- patchbot/base -- the latest release that all tickets are merged into
for testing
- patchbot/base_upstream -- temporary staging area for patchbot/base
- patchbot/ticket_upstream -- pristine clone of the ticket on trac
- patchbot/ticket_merged -- merge of patchbot/ticket_upstream into
patchbot/base
"""
merge_failure = False
is_safe = False
try:
os.chdir(sage_root)
info = scrape(ticket_id)
ensure_free_space(sage_root)
do_or_die("git checkout patchbot/base")
if ticket_id == 0:
do_or_die("git branch -f patchbot/ticket_upstream patchbot/base")
do_or_die("git branch -f patchbot/ticket_merged patchbot/base")
return
branch = info['git_branch']
repo = info['git_repo']
do_or_die("git fetch %s +%s:patchbot/ticket_upstream" % (repo, branch))
base = describe_branch('patchbot/ticket_upstream', tag_only=True)
do_or_die("git rev-list --left-right --count %s..patchbot/ticket_upstream" % base)
do_or_die("git branch -f patchbot/ticket_merged patchbot/base")
do_or_die("git checkout patchbot/ticket_merged")
try:
do_or_die("git merge -X patience patchbot/ticket_upstream")
except Exception:
do_or_die("git merge --abort")
merge_failure = True
raise
is_safe = inplace_safe()
if not is_safe:
if safe_only:
raise SkipTicket("unsafe")
tmp_dir = tempfile.mkdtemp(temp_build_suffix + str(ticket_id))
ensure_free_space(tmp_dir)
do_or_die("git clone . '{}'".format(tmp_dir))
os.chdir(tmp_dir)
os.symlink(os.path.join(sage_root, "upstream"), "upstream")
os.environ['SAGE_ROOT'] = tmp_dir
do_or_die("git branch -f patchbot/base remotes/origin/patchbot/base")
do_or_die("git branch -f patchbot/ticket_upstream remotes/origin/patchbot/ticket_upstream")
if use_ccache:
if not os.path.exists('logs'):
os.mkdir('logs')
do_or_die("./sage -i ccache")
except Exception as exn:
if merge_failure or (not is_safe):
raise
else:
raise ConfigException(exn.message)
def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
base = get_base(sage_root)
if ticket is None:
ticket = get_ticket(base=base, server=server, **conf)
else:
ticket = None, scrape(int(ticket))
if not ticket:
print "No more tickets."
if random.random() < 0.01:
cleanup(sage_root, server)
time.sleep(conf['idle'])
return
rating, ticket = ticket
print "\n" * 2
print "=" * 30, ticket['id'], "=" * 30
print ticket['title']
print "score", rating
print "\n" * 2
log_dir = sage_root + "/logs"
if not os.path.exists(log_dir):
os.mkdir(log_dir)
log = '%s/%s-log.txt' % (log_dir, ticket['id'])
report_ticket(server, ticket, status='Pending', base=base, machine=conf['machine'], user=conf['user'], log=None)
plugins_results = []
try:
with Tee(log, time=True, timeout=conf['timeout']):
t = Timer()
start_time = time.time()
state = 'started'
os.environ['MAKE'] = "make -j%s" % conf['parallelism']
os.environ['SAGE_ROOT'] = sage_root
# TODO: Ensure that sage-main is pristine.
pull_from_trac(sage_root, ticket['id'], force=True)
t.finish("Apply")
state = 'applied'
do_or_die('$SAGE_ROOT/sage -b %s' % ticket['id'])
t.finish("Build")
state = 'built'
working_dir = "%s/devel/sage-%s" % (sage_root, ticket['id'])
# Only the ones on this ticket.
patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):]
kwds = {
"original_dir": "%s/devel/sage-0" % sage_root,
"patched_dir": working_dir,
"patches": ["%s/devel/sage-%s/.hg/patches/%s" % (sage_root, ticket['id'], p) for p in patches if p],
}
for name, plugin in conf['plugins']:
try:
print plugin_boundary(name)
plugin(ticket, **kwds)
passed = True
except Exception:
traceback.print_exc()
passed = False
finally:
t.finish(name)
print plugin_boundary(name, end=True)
plugins_results.append((name, passed))
test_dirs = ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs]
if conf['parallelism'] > 1:
test_cmd = "-tp %s" % conf['parallelism']
else:
test_cmd = "-t"
do_or_die("$SAGE_ROOT/sage %s -sagenb %s" % (test_cmd, ' '.join(test_dirs)))
#do_or_die("$SAGE_ROOT/sage -t $SAGE_ROOT/devel/sage-%s/sage/rings/integer.pyx" % ticket['id'])
#do_or_die('sage -testall')
t.finish("Tests")
state = 'tested'
if not all(passed for name, passed in plugins_results):
state = 'failed_plugin'
print
t.print_all()
except Exception:
traceback.print_exc()
for _ in range(5):
try:
print "Reporting", ticket['id'], status[state]
report_ticket(server, ticket, status=status[state], base=base, machine=conf['machine'], user=conf['user'], log=log, plugins=plugins_results)
print "Done reporting", ticket['id']
break
except urllib2.HTTPError:
traceback.print_exc()
time.sleep(conf['idle'])
else:
print "Error reporting", ticket['id']
return status[state]
if not force:
raise Exception, "Duplicate patch: %s" % patch
old_patch = patch
while old_patch in series:
old_patch += '-old'
do_or_die('hg qrename %s %s' % (patch, old_patch))
try:
do_or_die('hg qimport %s' % url)
except Exception, exn:
time.sleep(30)
try:
do_or_die('hg qimport %s' % url)
except Exception, exn:
raise urllib2.HTTPError(exn)
do_or_die('hg qpush')
do_or_die('hg qapplied')
except:
os.system('hg qpop -a')
raise
def push_from_trac(sage_root, ticket, branch=None, force=None, interactive=None):
raise NotImplementedError
if __name__ == '__main__':
force = False
apply = False
for ticket in sys.argv[1:]:
if ticket == '-f':
def pull_from_trac(sage_root, ticket, branch=None, force=None, interactive=None):
# Should we set/unset SAGE_ROOT and SAGE_BRANCH here? Fork first?
if branch is None:
branch = str(ticket)
if not os.path.exists('%s/devel/sage-%s' % (sage_root, branch)):
do_or_die('%s/sage -b main' % (sage_root,))
do_or_die('%s/sage -clone %s' % (sage_root, branch))
os.chdir('%s/devel/sage-%s' % (sage_root, branch))
if interactive:
raise NotImplementedError
if not os.path.exists('.hg/patches'):
do_or_die('hg qinit')
series = []
elif not os.path.exists('.hg/patches/series'):
series = []
else:
series = open('.hg/patches/series').read().split('\n')
base = get_base(sage_root)
desired_series = []
seen_deps = []
def append_patch_list(ticket, dependency=False):
if ticket in seen_deps:
return
print "Looking at #%s" % ticket
seen_deps.append(ticket)
data = scrape(ticket)
if dependency and 'closed' in data['status']:
merged = data.get('merged')
if merged is None:
merged = data.get('milestone')
if merged is None or compare_version(merged, base) <= 0:
print "#%s already applied (%s <= %s)" % (ticket, merged, base)
return
if data['spkgs']:
raise NotImplementedError, "Spkgs not yet handled."
if data['depends_on']:
for dep in data['depends_on']:
if isinstance(dep, basestring) and '.' in dep:
if compare_version(base, dep) < 0:
raise ValueError, "%s < %s for %s" % (base, dep, ticket)
continue
append_patch_list(dep, dependency=True)
print "Patches for #%s:" % ticket
print " " + "\n ".join(data['patches'])
for patch in data['patches']:
patchfile, hash = patch.split('#')
desired_series.append((hash, patchfile, get_patch_url(ticket, patchfile)))
append_patch_list(ticket)
ensure_safe(series)
ensure_safe(patch for hash, patch, url in desired_series)
last_good_patch = '-a'
to_push = list(desired_series)
for series_patch, (hash, patch, url) in zip(series, desired_series):
if not series_patch:
break
next_hash = digest(open('.hg/patches/%s' % series_patch).read())
# print next_hash, hash, series_patch
if next_hash == hash:
to_push.pop(0)
last_good_patch = series_patch
else:
break
try:
if last_good_patch != '-a':
# In case it's not yet pushed...
if last_good_patch not in os.popen2('hg qapplied')[1].read().split('\n'):
do_or_die('hg qpush %s' % last_good_patch)
do_or_die('hg qpop %s' % last_good_patch)
for hash, patch, url in to_push:
if patch in series:
if not force:
raise Exception, "Duplicate patch: %s" % patch
old_patch = patch
while old_patch in series:
old_patch += '-old'
do_or_die('hg qrename %s %s' % (patch, old_patch))
try:
do_or_die('hg qimport %s' % url)
except Exception, exn:
time.sleep(30)
try:
do_or_die('hg qimport %s' % url)
except Exception, exn:
raise urllib2.HTTPError(exn)
do_or_die('hg qpush')
def check_spkg(self, spkg):
temp_dir = None
try:
if '#' in spkg:
spkg = spkg.split('#')[0]
basename = os.path.basename(spkg)
temp_dir = tempfile.mkdtemp()
local_spkg = os.path.join(temp_dir, basename)
do_or_die("wget --progress=dot:mega -O %s %s" % (local_spkg, spkg))
do_or_die("tar xf %s -C %s" % (local_spkg, temp_dir))
print
print "Sha1", basename, sha1file(local_spkg)
print
print "Checking repo status."
do_or_die("cd %s; hg diff; echo $?" % local_spkg[:-5])
print
print
print "Comparing to previous spkg."
# Compare to the current version.
base = basename.split('-')[0] # the reset is the version
old_path = old_url = listing = None
if False:
# There seems to be a bug...
# File "/data/sage/sage-5.5/local/lib/python2.7/site-packages/pexpect.py", line 1137, in which
# if os.access (filename, os.X_OK) and not os.path.isdir(f):
import pexpect
p = pexpect.spawn("%s/sage" % self.sage_root, ['-i', '--info', base])
while True:
index = p.expect([
r"Found package %s in (\S+)" % base,
r">>> Checking online list of (\S+) packages.",
r">>> Found (%s-\S+)" % base,
r"Error: could not find a package"])
if index == 0:
old_path = "$SAGE_ROOT/" + p.match.group(1)
break
elif index == 1:
listing = p.match.group(2)
elif index == 2:
old_url = "http://www.sagemath.org/packages/%s/%s.spkg" % (listing, p.match.group(1))
break
else:
print "No previous match."
break
else:
p = subprocess.Popen(r"%s/sage -i --info %s" % (self.sage_root, base),
shell=True, stdout=subprocess.PIPE)
for line in p.communicate()[0].split('\n'):
m = re.match(r"Found package %s in (\S+)" % base, line)
if m:
old_path = os.path.join(self.sage_root, m.group(1))
break
m = re.match(r">>> Checking online list of (\S+) packages.", line)
if m:
listing = m.group(1)
m = re.match(r">>> Found (%s-\S+)" % base, line)
if m:
old_url = "http://www.sagemath.org/packages/%s/%s.spkg" % (listing, m.group(1))
break
if not old_path and not old_url:
print "Unable to locate existing package %s." % base
if old_path is not None and old_path.startswith('/attachment/'):
old_url = 'http://trac.sagemath.org/sage_trac' + old_path
if old_url is not None:
old_basename = os.path.basename(old_url)
old_path = os.path.join(temp_dir, old_basename)
if not os.path.exists(old_path):
do_or_die("wget --progress=dot:mega %s -O %s" % (old_url, old_path))
if old_path is not None:
old_basename = os.path.basename(old_path)
if old_basename == basename:
print "PACKAGE NOT RENAMED"
else:
do_or_die("tar xf %s -C %s" % (old_path, temp_dir))
print '\n\n', '-' * 20
do_or_die("diff -N -u -r -x src -x .hg %s/%s %s/%s; echo $?" % (temp_dir, old_basename[:-5], temp_dir, basename[:-5]))
print '\n\n', '-' * 20
do_or_die("diff -q -r %s/%s/src %s/%s/src; echo $?" % (temp_dir, old_basename[:-5], temp_dir, basename[:-5]))
print
print "-" * 20
if old_path:
do_or_die("head -n 100 %s/SPKG.txt" % local_spkg[:-5])
else:
do_or_die("cat %s/SPKG.txt" % local_spkg[:-5])
finally:
if temp_dir and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
def main(args):
global conf
# Most configuration is done in the config file, which is reread between
# each ticket for live configuration of the patchbot.
parser = OptionParser()
parser.add_option("--config", dest="config")
parser.add_option("--sage-root", dest="sage_root", default=os.environ.get('SAGE_ROOT'))
parser.add_option("--server", dest="server", default="http://patchbot.sagemath.org/")
parser.add_option("--count", dest="count", default=1000000)
parser.add_option("--ticket", dest="ticket", default=None)
parser.add_option("--list", dest="list", default=False)
parser.add_option("--full", action="store_true", dest="full", default=False)
parser.add_option("--skip-base", action="store_true", dest="skip_base", default=False)
parser.add_option("--dry-run", action="store_true", dest="dry_run", default=False)
parser.add_option("--plugin-only", action="store_true", dest="plugin_only", default=False)
(options, args) = parser.parse_args(args)
conf_path = options.config and os.path.abspath(options.config)
if options.ticket:
tickets = [int(t) for t in options.ticket.split(',')]
count = len(tickets)
else:
tickets = None
count = int(options.count)
patchbot = Patchbot(os.path.abspath(options.sage_root), options.server, conf_path, dry_run=options.dry_run, plugin_only=options.plugin_only)
conf = patchbot.get_config()
if options.list:
count = sys.maxint if options.list is "True" else int(options.list)
print "Getting ticket list..."
for ix, (score, ticket) in enumerate(patchbot.get_ticket_list()):
if ix >= count:
break
print score, '\t', ticket['id'], '\t', ticket['title']
if options.full:
print ticket
print
sys.exit(0)
if options.sage_root == os.environ.get('SAGE_ROOT'):
print "WARNING: Do not use this copy of sage while the patchbot is running."
if conf['use_ccache']:
do_or_die("'%s'/sage -i ccache" % options.sage_root)
# If we rebuild the (same) compiler we still want to share the cache.
os.environ['CCACHE_COMPILERCHECK'] = '%compiler% --version'
if not options.skip_base:
patchbot.check_base()
def good(report):
return report['machine'] == conf['machine'] and report['status'] == 'TestsPassed'
if options.plugin_only or not any(good(report) for report in patchbot.current_reports(0)):
res = patchbot.test_a_ticket(0)
if res not in ('TestsPassed', 'PluginOnly'):
print "\n\n"
while True:
print "Failing tests in your install: %s. Continue anyways? [y/N] " % res
ans = sys.stdin.readline().lower().strip()
if ans == '' or ans[0] == 'n':
sys.exit(1)
elif ans[0] == 'y':
break
for k in range(count):
try:
if tickets:
ticket = tickets.pop(0)
else:
ticket = None
conf = patchbot.reload_config()
if check_time_of_day(conf['time_of_day']):
if not patchbot.check_base():
patchbot.test_a_ticket(0)
patchbot.test_a_ticket(ticket)
else:
print "Idle."
time.sleep(conf['idle'])
except Exception:
traceback.print_exc()
time.sleep(conf['idle'])
def test_a_ticket(self, ticket=None):
self.reload_config()
if ticket is None:
ticket = self.get_ticket()
else:
ticket = None, scrape(int(ticket))
if not ticket:
print "No more tickets."
time.sleep(self.config['idle'])
return
rating, ticket = ticket
print "\n" * 2
print "=" * 30, ticket['id'], "=" * 30
print ticket['title']
print "score", rating
print "\n" * 2
self.log_dir = self.sage_root + "/logs/patchbot"
if not os.path.exists(self.log_dir):
os.makedirs(self.log_dir)
log = '%s/%s-log.txt' % (self.log_dir, ticket['id'])
history = open("%s/history.txt" % self.log_dir, "a")
history.write("%s %s\n" % (datetime(), ticket['id']))
history.close()
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=log)
plugins_results = []
try:
t = Timer()
with Tee(log, time=True, timeout=self.config['timeout'], timer=t):
print "Sage Patchbot", patchbot_version.get_version()
if ticket['spkgs']:
state = 'spkg'
print "\n".join(ticket['spkgs'])
print
for spkg in ticket['spkgs']:
print
print '+' * 10, spkg, '+' * 10
print
try:
self.check_spkg(spkg)
except Exception:
traceback.print_exc()
t.finish(spkg)
if not ticket['spkgs']:
state = 'started'
os.environ['MAKE'] = "make -j%s" % self.config['parallelism']
os.environ['SAGE_ROOT'] = self.sage_root
os.environ['GIT_AUTHOR_NAME'] = os.environ['GIT_COMMITTER_NAME'] = 'patchbot'
os.environ['GIT_AUTHOR_EMAIL'] = os.environ['GIT_COMMITTER_EMAIL'] = 'patchbot@localhost'
os.environ['GIT_AUTHOR_DATE'] = os.environ['GIT_COMMITTER_DATE'] = '1970-01-01T00:00:00'
pull_from_trac(self.sage_root, ticket['id'], force=True, use_ccache=self.config['use_ccache'])
t.finish("Apply")
state = 'applied'
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=log, pending_status=state)
do_or_die("$MAKE")
t.finish("Build")
state = 'built'
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=log, pending_status=state)
# TODO: Exclude dependencies.
patch_dir = tempfile.mkdtemp()
if ticket['id'] != 0:
do_or_die("git format-patch -o '%s' patchbot/base..patchbot/ticket_merged" % patch_dir)
kwds = {
"original_dir": self.sage_root,
"patched_dir": os.getcwd(),
"patches": [os.path.join(patch_dir, p) for p in os.listdir(patch_dir)],
"sage_binary": os.path.join(os.getcwd(), 'sage'),
"dry_run": self.dry_run,
}
for name, plugin in self.config['plugins']:
try:
if ticket['id'] != 0 and os.path.exists(os.path.join(self.log_dir, '0', name)):
baseline = pickle.load(open(os.path.join(self.log_dir, '0', name)))
else:
baseline = None
print plugin_boundary(name)
do_or_die("git checkout patchbot/ticket_merged")
res = plugin(ticket, is_git=True, baseline=baseline, **kwds)
passed = True
except Exception:
traceback.print_exc()
passed = False
res = None
finally:
if isinstance(res, PluginResult):
if res.baseline is not None:
plugin_dir = os.path.join(self.log_dir, str(ticket['id']))
if not os.path.exists(plugin_dir):
os.mkdir(plugin_dir)
pickle.dump(res.baseline, open(os.path.join(plugin_dir, name), 'w'))
passed = res.status == PluginResult.Passed
print name, res.status
plugins_results.append((name, passed, res.data))
else:
plugins_results.append((name, passed, None))
t.finish(name)
print plugin_boundary(name, end=True)
plugins_passed = all(passed for (name, passed, data) in plugins_results)
self.report_ticket(ticket, status='Pending', log=log, pending_status='plugins_passed' if plugins_passed else 'plugins_failed')
if self.plugin_only:
state = 'plugins' if plugins_passed else 'plugins_failed'
else:
if self.dry_run:
test_target = "$SAGE_ROOT/src/sage/misc/a*.py"
# TODO: Remove
test_target = "$SAGE_ROOT/src/sage/doctest/*.py"
else:
test_target = "--all --long"
if self.config['parallelism'] > 1:
test_cmd = "-tp %s" % self.config['parallelism']
else:
test_cmd = "-t"
do_or_die("$SAGE_ROOT/sage %s %s" % (test_cmd, test_target))
t.finish("Tests")
state = 'tested'
if not plugins_passed:
state = 'tests_passed_plugins_failed'
except (urllib2.HTTPError, socket.error):
# Don't report failure because the network/trac died...
print
t.print_all()
traceback.print_exc()
state = 'network_error'
except Exception:
traceback.print_exc()
for _ in range(5):
try:
print "Reporting", ticket['id'], status[state]
self.report_ticket(ticket, status=status[state], log=log, plugins=plugins_results, dry_run=self.dry_run)
print "Done reporting", ticket['id']
break
except IOError:
traceback.print_exc()
time.sleep(self.config['idle'])
else:
print "Error reporting", ticket['id']
maybe_temp_root = os.environ['SAGE_ROOT']
if maybe_temp_root.endswith("-sage-git-temp-%s" % ticket['id']):
shutil.rmtree(maybe_temp_root)
return status[state]
def check_base(self):
os.chdir(self.sage_root)
try:
do_or_die("git checkout patchbot/base")
except Exception:
do_or_die("git checkout -b patchbot/base")
do_or_die("git fetch %s +%s:patchbot/base_upstream" % (self.config['base_repo'], self.config['base_branch']))
only_in_base = int(subprocess.check_output(["git", "rev-list", "--count", "patchbot/base_upstream..patchbot/base"]))
only_in_upstream = int(subprocess.check_output(["git", "rev-list", "--count", "patchbot/base..patchbot/base_upstream"]))
if (only_in_base > 0
or only_in_upstream > self.config['max_behind_commits']
or (only_in_upstream > 0 and time.time() - self.last_pull < self.config['max_behind_days'] * 60 * 60 * 24)):
do_or_die("git checkout patchbot/base_upstream")
do_or_die("git branch -f patchbot/base patchbot/base_upstream")
do_or_die("git checkout patchbot/base")
self.last_pull = time.time()
self.behind_base = {}
return False
return True
def pull_from_trac(sage_root, ticket, branch=None, force=None, interactive=None, inplace=None, use_ccache=False):
# There are four branches at play here:
# patchbot/base -- the latest release that all tickets are merged into for testing
# patchbot/base_upstream -- temporary staging area for patchbot/base
# patchbot/ticket_upstream -- pristine clone of the ticket on trac
# patchbot/ticket_merged -- merge of patchbot/ticket_upstream into patchbot/base
ticket_id = ticket
info = scrape(ticket_id)
os.chdir(sage_root)
do_or_die("git checkout patchbot/base")
if ticket_id == 0:
do_or_die("git branch -f patchbot/ticket_upstream patchbot/base")
do_or_die("git branch -f patchbot/ticket_merged patchbot/base")
return
branch = info['git_branch']
repo = info['git_repo']
do_or_die("git fetch %s +%s:patchbot/ticket_upstream" % (repo, branch))
do_or_die("git rev-list --left-right --count patchbot/base..patchbot/ticket_upstream")
do_or_die("git branch -f patchbot/ticket_merged patchbot/base")
do_or_die("git checkout patchbot/ticket_merged")
try:
do_or_die("git merge -X patience patchbot/ticket_upstream")
except Exception:
do_or_die("git merge --abort")
raise
if not inplace_safe():
tmp_dir = tempfile.mkdtemp("-sage-git-temp-%s" % ticket_id)
do_or_die("git clone . '%s'" % tmp_dir)
os.chdir(tmp_dir)
os.symlink(os.path.join(sage_root, "upstream"), "upstream")
os.environ['SAGE_ROOT'] = tmp_dir
do_or_die("git branch -f patchbot/base remotes/origin/patchbot/base")
do_or_die("git branch -f patchbot/ticket_upstream remotes/origin/patchbot/ticket_upstream")
if use_ccache:
do_or_die("./sage -i ccache")
def test_a_ticket(self, ticket=None):
self.reload_config()
if ticket is None:
ticket = self.get_ticket()
else:
ticket = None, scrape(int(ticket))
if not ticket:
print "No more tickets."
time.sleep(conf['idle'])
return
rating, ticket = ticket
print "\n" * 2
print "=" * 30, ticket['id'], "=" * 30
print ticket['title']
print "score", rating
print "\n" * 2
log_dir = self.sage_root + "/logs"
if not os.path.exists(log_dir):
os.mkdir(log_dir)
log = '%s/%s-log.txt' % (log_dir, ticket['id'])
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=None)
plugins_results = []
try:
with Tee(log, time=True, timeout=self.config['timeout']):
t = Timer()
start_time = time.time()
if ticket['spkgs']:
state = 'spkg'
print "\n".join(ticket['spkgs'])
print
for spkg in ticket['spkgs']:
print
print '+' * 10, spkg, '+' * 10
print
try:
self.check_spkg(spkg)
except Exception:
traceback.print_exc()
t.finish(spkg)
else:
state = 'started'
os.environ['MAKE'] = "make -j%s" % self.config['parallelism']
os.environ['SAGE_ROOT'] = self.sage_root
# TODO: Ensure that sage-main is pristine.
pull_from_trac(self.sage_root, ticket['id'], force=True)
t.finish("Apply")
state = 'applied'
do_or_die('$SAGE_ROOT/sage -b %s' % ticket['id'])
t.finish("Build")
state = 'built'
working_dir = "%s/devel/sage-%s" % (self.sage_root, ticket['id'])
# Only the ones on this ticket.
patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):]
kwds = {
"original_dir": "%s/devel/sage-0" % self.sage_root,
"patched_dir": working_dir,
"patches": ["%s/devel/sage-%s/.hg/patches/%s" % (self.sage_root, ticket['id'], p) for p in patches if p],
"sage_binary": os.path.join(self.sage_root, 'sage')
}
for name, plugin in self.config['plugins']:
try:
if ticket['id'] != 0 and os.path.exists(os.path.join(log_dir, '0', name)):
baseline = pickle.load(open(os.path.join(log_dir, '0', name)))
else:
baseline = None
print plugin_boundary(name)
res = plugin(ticket, baseline=baseline, **kwds)
passed = True
except Exception:
traceback.print_exc()
passed = False
res = None
finally:
if isinstance(res, PluginResult):
if res.baseline is not None:
plugin_dir = os.path.join(log_dir, str(ticket['id']))
if not os.path.exists(plugin_dir):
os.mkdir(plugin_dir)
pickle.dump(res.baseline, open(os.path.join(plugin_dir, name), 'w'))
passed = res.status == PluginResult.Passed
print name, res.status
plugins_results.append((name, passed, res.data))
else:
plugins_results.append((name, passed, None))
t.finish(name)
print plugin_boundary(name, end=True)
plugins_passed = all(passed for (name, passed, data) in plugins_results)
if self.plugin_only:
state = 'plugins' if plugins_passed else 'plugins_failed'
else:
if self.dry_run:
test_dirs = ["$SAGE_ROOT/devel/sage-%s/sage/misc/a*.py" % (ticket['id'])]
else:
test_dirs = ["--sagenb"] + ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs]
if conf['parallelism'] > 1:
test_cmd = "-tp %s" % conf['parallelism']
else:
test_cmd = "-t"
do_or_die("$SAGE_ROOT/sage %s %s" % (test_cmd, ' '.join(test_dirs)))
t.finish("Tests")
state = 'tested'
if not plugins_passed:
state = 'tests_passed_plugins_failed'
print
t.print_all()
except urllib2.HTTPError:
# Don't report failure because the network/trac died...
traceback.print_exc()
return 'Pending'
except Exception:
traceback.print_exc()
for _ in range(5):
try:
print "Reporting", ticket['id'], status[state]
if not self.dry_run:
self.report_ticket(ticket, status=status[state], log=log, plugins=plugins_results)
print "Done reporting", ticket['id']
break
except urllib2.HTTPError:
traceback.print_exc()
time.sleep(conf['idle'])
else:
print "Error reporting", ticket['id']
if not conf['keep_open_branches'] and str(ticket['id']) != '0' and not ticket['spkgs']:
shutil.rmtree(os.path.join(self.sage_root, "devel", "sage-%s" %ticket['id']))
return status[state]
def main(args):
"""
Most configuration is done in the config file, which is reread between
each ticket for live configuration of the patchbot.
"""
global conf
parser = OptionParser()
parser.add_option("--config", dest="config")
parser.add_option("--sage-root", dest="sage_root", default=os.environ.get('SAGE_ROOT'))
parser.add_option("--server", dest="server", default="http://patchbot.sagemath.org/")
parser.add_option("--count", dest="count", default=1000000)
parser.add_option("--ticket", dest="ticket", default=None)
parser.add_option("--list", dest="list", default=False)
parser.add_option("--full", action="store_true", dest="full", default=False)
parser.add_option("--skip-base", action="store_true", dest="skip_base", default=False)
parser.add_option("--dry-run", action="store_true", dest="dry_run", default=False)
parser.add_option("--plugin-only", action="store_true", dest="plugin_only", default=False)
parser.add_option("--cleanup", action="store_true", dest="cleanup", default=False)
parser.add_option("--safe-only", action="store_true", dest="safe_only", default=False)
parser.add_option("--interactive", action="store_true", dest="interactive", default=False)
(options, args) = parser.parse_args(args)
conf_path = options.config and os.path.abspath(options.config)
if options.ticket:
tickets = [int(t) for t in options.ticket.split(',')]
count = len(tickets)
else:
tickets = None
count = int(options.count)
patchbot = Patchbot(os.path.abspath(options.sage_root), options.server, conf_path, dry_run=options.dry_run, plugin_only=options.plugin_only, options=options)
conf = patchbot.get_config()
if options.list:
count = sys.maxint if options.list is "True" else int(options.list)
print "Getting ticket list..."
for ix, (score, ticket) in enumerate(patchbot.get_ticket_list()):
if ix >= count:
break
print score, '\t', ticket['id'], '\t', ticket['title']
if options.full:
print ticket
print
sys.exit(0)
if options.sage_root == os.environ.get('SAGE_ROOT'):
print "WARNING: Do not use this copy of sage while the patchbot is running."
ensure_free_space(options.sage_root)
log_dir = options.sage_root + "/logs"
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Make sure this file is writable.
handle = open(os.path.join(log_dir, 'install.log'), 'a')
handle.close()
if conf['use_ccache']:
do_or_die("'%s'/sage -i ccache" % options.sage_root, exn_class=ConfigException)
# If we rebuild the (same) compiler we still want to share the cache.
os.environ['CCACHE_COMPILERCHECK'] = '%compiler% --version'
if not options.skip_base:
patchbot.check_base()
def good(report):
return report['machine'] == conf['machine'] and report['status'] == 'TestsPassed'
if options.plugin_only or not any(good(report) for report in patchbot.current_reports(0)):
res = patchbot.test_a_ticket(0)
if res not in ('TestsPassed', 'PluginOnly'):
print "\n\n"
print "Current base:", conf['base_repo'], conf['base_branch']
if not options.interactive:
print "Failing tests in your base install: exiting."
sys.exit(1)
while True:
print "Failing tests in your base install: %s. Continue anyways? [y/N] " % res
try:
ans = sys.stdin.readline().lower().strip()
except IOError:
# Might not be interactive.
print "Non interactive, not continuing."
ans = 'n'
if ans == '' or ans[0] == 'n':
sys.exit(1)
elif ans[0] == 'y':
break
for k in range(count):
if options.cleanup:
for path in glob.glob(os.path.join(tempfile.gettempdir(), "*%s*" % temp_build_suffix)):
print "Cleaning up ", path
shutil.rmtree(path)
try:
if tickets:
ticket = tickets.pop(0)
else:
ticket = None
conf = patchbot.reload_config()
if check_time_of_day(conf['time_of_day']):
if not patchbot.check_base():
patchbot.test_a_ticket(0)
patchbot.test_a_ticket(ticket)
else:
print "Idle."
time.sleep(conf['idle'])
except Exception:
traceback.print_exc()
time.sleep(conf['idle'])