def lookup_ticket(self, id):
path = "ticket/?" + urllib.urlencode({'raw': True, 'query': json.dumps({'id': id})})
res = self.load_json_from_server(path)
if res:
return res[0]
else:
return scrape(id)
def lookup_ticket(server, id):
url = server + "/ticket/?" + urllib.urlencode({'raw': True, 'query': json.dumps({'id': id})})
res = json.load(urllib2.urlopen(url))
if res:
return res[0]
else:
return scrape(id)
def post_report(ticket_id):
"""
Posting a report to the database of reports.
"""
try:
ticket = tickets.find_one({'id': ticket_id})
if ticket is None:
ticket = scrape(ticket_id)
if 'reports' not in ticket:
ticket['reports'] = []
report = json.loads(request.form.get('report'))
assert (isinstance(report, dict)), "report is not a dict"
for fld in ['status', 'spkgs', 'base', 'machine', 'time']:
assert (fld in report), "{} missing in report".format(fld)
machine_name = report['machine'][-1]
if machine_name in BLACKLIST:
msg = 'machine {} is blacklisted'.format(machine_name)
raise RuntimeError(msg)
prune_pending(ticket, report['machine'])
ticket['reports'].append(report)
db.logs.put(request.files.get('log'), _id=log_name(ticket_id, report))
if 'retry' in ticket:
ticket['retry'] = False
ticket['last_activity'] = now_str()
db.save_ticket(ticket)
return "ok (report successfully posted)"
except:
traceback.print_exc()
return "error in posting the report"
def render_ticket_status_svg(ticket):
"""
Return the svg status image for the given ticket.
"""
try:
if 'fast' in request.args:
info = tickets.find_one({'id': ticket})
else:
info = scrape(ticket, db=db)
except:
info = tickets.find_one({'id': ticket})
if 'base' in request.args:
base = request.args.get('base')
else:
base = latest_version(info.get('reports', []))
status = get_ticket_status(info, base=base)[1] # single status
path = status_image_path(status, image_type='svg')
# with no base
response = make_response(open(path).read())
response.headers['Content-type'] = 'image/svg+xml'
response.headers['Cache-Control'] = 'no-cache'
return response
def render_ticket_status(ticket):
"""
Return the status image for the given ticket.
This now only renders a png image with the base.
"""
try:
if 'fast' in request.args:
info = tickets.find_one({'id': ticket})
else:
info = scrape(ticket, db=db)
except:
info = tickets.find_one({'id': ticket})
if 'base' in request.args:
base = request.args.get('base')
else:
base = latest_version(info.get('reports', []))
# status = get_ticket_status(info, base=base)[2] # composite status
if 'fast' in request.args:
display_base = None
else:
display_base = base
response = make_response(create_base_image(base=display_base))
response.headers['Content-type'] = 'image/png'
response.headers['Cache-Control'] = 'no-cache'
return response
def render_ticket_status(ticket):
try:
info = trac.scrape(ticket, db=db)
except:
info = tickets.find_one({'id': ticket})
if 'reports' in info:
base = latest_version(current_reports(info))
else:
base = None
status = get_ticket_status(info, base=base)[2]
response = make_response(create_status_image(status, base=base))
response.headers['Content-type'] = 'image/png'
response.headers['Cache-Control'] = 'no-cache'
return response
def render_ticket(ticket):
try:
info = trac.scrape(ticket, db=db, force='force' in request.args)
except:
info = tickets.find_one({'id': ticket})
if info is None:
return "No such ticket."
if 'kick' in request.args:
info['retry'] = True
db.save_ticket(info)
if 'reports' in info:
info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
else:
info['reports'] = []
old_reports = list(info['reports'])
patchbot.prune_pending(info)
if old_reports != info['reports']:
db.save_ticket(info)
def format_info(info):
new_info = {}
for key, value in info.items():
if key == 'patches':
new_info['patches'] = format_patches(ticket, value)
elif key == 'reports' or key == 'pending':
pass
elif key == 'depends_on':
new_info[key] = ', '.join("<a href='/ticket/%s'>%s</a>" % (a, a) for a in value)
elif key == 'authors':
new_info[key] = ', '.join("<a href='/ticket/?author=%s'>%s</a>" % (a,a) for a in value)
elif key == 'participants':
new_info[key] = ', '.join("<a href='/ticket/?participant=%s'>%s</a>" % (a,a) for a in value)
elif isinstance(value, list):
new_info[key] = ', '.join(value)
elif key not in ('id', '_id'):
new_info[key] = value
return new_info
def preprocess_reports(all):
for item in all:
if 'patches' in item:
required = info['depends_on'] + info['patches']
item['patch_list'] = format_patches(ticket, item['patches'], item.get('deps'), required)
if item['base'] != base:
item['base'] = "<span style='color: red'>%s</span>" % item['base']
if 'time' in item:
item['log'] = log_name(info['id'], item)
yield item
return render_template("ticket.html", reports=preprocess_reports(info['reports']), ticket=ticket, info=format_info(info), status=get_ticket_status(info, base=base)[2])
def render_ticket_status(ticket):
try:
if 'fast' in request.args:
info = tickets.find_one({'id': ticket})
else:
info = trac.scrape(ticket, db=db)
except:
info = tickets.find_one({'id': ticket})
if 'base' in request.args:
base = request.args.get('base')
else:
base = latest_version(info['reports'] or [])
status = get_ticket_status(info, base=base)[2]
if 'fast' in request.args:
display_base = None
else:
display_base = base
response = make_response(create_status_image(status, base=display_base))
response.headers['Content-type'] = 'image/png'
response.headers['Cache-Control'] = 'no-cache'
return response
def post_report(ticket_id):
try:
ticket = db.lookup_ticket(ticket_id)
if ticket is None:
ticket = trac.scrape(ticket_id)
if 'reports' not in ticket:
ticket['reports'] = []
report = json.loads(request.form.get('report'))
assert isinstance(report, dict)
for fld in ['status', 'patches', 'spkgs', 'base', 'machine', 'time']:
assert fld in report
patchbot.prune_pending(ticket, report['machine'])
ticket['reports'].append(report)
db.logs.put(request.files.get('log'), _id=log_name(ticket_id, report))
if 'retry' in ticket:
ticket['retry'] = False
ticket['last_activity'] = now_str()
db.save_ticket(ticket)
return "ok"
except:
traceback.print_exc()
return "error"
def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
base = get_base(sage_root)
if ticket is None:
ticket = get_ticket(base=base, server=server, **conf)
else:
ticket = None, scrape(int(ticket))
if not ticket:
print "No more tickets."
if random.random() < 0.01:
cleanup(sage_root, server)
time.sleep(conf['idle'])
return
rating, ticket = ticket
print "\n" * 2
print "=" * 30, ticket['id'], "=" * 30
print ticket['title']
print "score", rating
print "\n" * 2
log_dir = sage_root + "/logs"
if not os.path.exists(log_dir):
os.mkdir(log_dir)
log = '%s/%s-log.txt' % (log_dir, ticket['id'])
report_ticket(server, ticket, status='Pending', base=base, machine=conf['machine'], user=conf['user'], log=None)
plugins_results = []
try:
with Tee(log, time=True, timeout=conf['timeout']):
t = Timer()
start_time = time.time()
state = 'started'
os.environ['MAKE'] = "make -j%s" % conf['parallelism']
os.environ['SAGE_ROOT'] = sage_root
# TODO: Ensure that sage-main is pristine.
pull_from_trac(sage_root, ticket['id'], force=True)
t.finish("Apply")
state = 'applied'
do_or_die('$SAGE_ROOT/sage -b %s' % ticket['id'])
t.finish("Build")
state = 'built'
working_dir = "%s/devel/sage-%s" % (sage_root, ticket['id'])
# Only the ones on this ticket.
patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):]
kwds = {
"original_dir": "%s/devel/sage-0" % sage_root,
"patched_dir": working_dir,
"patches": ["%s/devel/sage-%s/.hg/patches/%s" % (sage_root, ticket['id'], p) for p in patches if p],
}
for name, plugin in conf['plugins']:
try:
print plugin_boundary(name)
plugin(ticket, **kwds)
passed = True
except Exception:
traceback.print_exc()
passed = False
finally:
t.finish(name)
print plugin_boundary(name, end=True)
plugins_results.append((name, passed))
test_dirs = ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs]
if conf['parallelism'] > 1:
test_cmd = "-tp %s" % conf['parallelism']
else:
test_cmd = "-t"
do_or_die("$SAGE_ROOT/sage %s -sagenb %s" % (test_cmd, ' '.join(test_dirs)))
#do_or_die("$SAGE_ROOT/sage -t $SAGE_ROOT/devel/sage-%s/sage/rings/integer.pyx" % ticket['id'])
#do_or_die('sage -testall')
t.finish("Tests")
state = 'tested'
if not all(passed for name, passed in plugins_results):
state = 'failed_plugin'
print
t.print_all()
except Exception:
traceback.print_exc()
for _ in range(5):
try:
print "Reporting", ticket['id'], status[state]
report_ticket(server, ticket, status=status[state], base=base, machine=conf['machine'], user=conf['user'], log=log, plugins=plugins_results)
print "Done reporting", ticket['id']
break
except urllib2.HTTPError:
traceback.print_exc()
time.sleep(conf['idle'])
else:
print "Error reporting", ticket['id']
return status[state]
def render_ticket(ticket):
try:
info = trac.scrape(ticket, db=db, force='force' in request.args)
except:
info = tickets.find_one({'id': ticket})
if info is None:
return "No such ticket."
if 'kick' in request.args:
info['retry'] = True
db.save_ticket(info)
if 'reports' in info:
info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
else:
info['reports'] = []
base_reports = base_reports_by_machine_and_base()
old_reports = list(info['reports'])
patchbot.prune_pending(info)
if old_reports != info['reports']:
db.save_ticket(info)
def format_info(info):
new_info = {}
for key, value in info.items():
if key == 'patches':
new_info['patches'] = format_patches(ticket, value)
elif key == 'reports' or key == 'pending':
pass
elif key == 'depends_on':
deps_status = {}
for dep in tickets.find({'id': {'$in': [int(a) for a in value]}}, ['status', 'id']):
if 'closed' in dep['status']:
dep['style'] = 'text-decoration: line-through'
else:
dep['style'] = ''
deps_status[dep['id']] = dep
new_info[key] = ', '.join("<img src='/ticket/%s/status.png?fast' height=16><a href='/ticket/%s' style='%s'>%s</a>" % (a, a, deps_status[a]['style'], a) for a in value)
elif key == 'authors':
new_info[key] = ', '.join("<a href='/ticket/?author=%s'>%s</a>" % (a,a) for a in value)
elif key == 'participants':
new_info[key] = ', '.join("<a href='/ticket/?participant=%s'>%s</a>" % (a,a) for a in value)
elif isinstance(value, list):
new_info[key] = ', '.join(value)
elif key not in ('id', '_id'):
new_info[key] = value
return new_info
def preprocess_reports(all):
for item in all:
base_report = base_reports.get(item['base'] + "/" + "/".join(item['machine']), base_reports.get(item['base']))
if base_report:
item['base_log'] = urllib.quote(log_name(0, base_report))
if 'patches' in item:
required = info['depends_on'] + info['patches']
item['patch_list'] = format_patches(ticket, item['patches'], item.get('deps'), required)
item['raw_base'] = item['base']
if item['base'] != base:
item['base'] = "<span style='color: red'>%s</span>" % item['base']
if 'time' in item:
item['log'] = log_name(info['id'], item)
yield item
def normalize_plugin(plugin):
while len(plugin) < 3:
plugin.append(None)
return plugin
return render_template("ticket.html", reports=preprocess_reports(info['reports']), ticket=ticket, info=format_info(info), status=get_ticket_status(info, base=base)[2], normalize_plugin=normalize_plugin)
def render_ticket(ticket):
try:
info = trac.scrape(ticket, db=db, force='force' in request.args)
except:
info = tickets.find_one({'id': ticket})
if info is None:
return "No such ticket."
if 'kick' in request.args:
info['retry'] = True
db.save_ticket(info)
if 'reports' in info:
info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
else:
info['reports'] = []
base_reports = base_reports_by_machine_and_base()
old_reports = list(info['reports'])
patchbot.prune_pending(info)
if old_reports != info['reports']:
db.save_ticket(info)
def format_info(info):
new_info = {}
for key, value in info.items():
if key == 'patches':
new_info['patches'] = format_patches(ticket, value)
elif key == 'reports' or key == 'pending':
pass
elif key == 'depends_on':
deps_status = {}
def is_int(a):
try:
int(a)
return True
except ValueError:
return False
for dep in tickets.find({'id': {'$in': [int(a) for a in value if is_int(a)]}}, ['status', 'id']):
if 'closed' in dep['status']:
dep['style'] = 'text-decoration: line-through'
else:
dep['style'] = ''
deps_status[dep['id']] = dep
new_info[key] = ', '.join("<img src='/ticket/%s/status.png?fast' height=16><a href='/ticket/%s' style='%s'>%s</a>" % (a, a, deps_status[a]['style'], a) for a in value)
elif key == 'authors':
new_info[key] = ', '.join("<a href='/ticket/?author=%s'>%s</a>" % (a,a) for a in value)
elif key == 'participants':
new_info[key] = ', '.join("<a href='/ticket/?participant=%s'>%s</a>" % (a,a) for a in value)
elif isinstance(value, list):
new_info[key] = ', '.join(value)
elif key not in ('id', '_id'):
new_info[key] = value
return new_info
base = latest_base()
def format_git_describe(res):
if res:
if '-' in res:
tag, commits = res.split('-')[:2]
return "%s + %s commits" % (tag, commits)
elif 'commits' in res:
# old style
return res
else:
return res + " + 0 commits"
else:
return '?'
def preprocess_reports(all):
for item in all:
base_report = base_reports.get(item['base'] + "/" + "/".join(item['machine']), base_reports.get(item['base']))
if base_report:
item['base_log'] = urllib.quote(log_name(0, base_report))
if 'patches' in item:
required = info['depends_on'] + info['patches']
item['patch_list'] = format_patches(ticket, item['patches'], item.get('deps'), required)
if 'git_base' in item:
git_log = item.get('git_log')
item['git_log_len'] = '?' if git_log is None else len(git_log)
item['raw_base'] = item['base']
if compare_version(item['base'], base) < 0:
item['base'] = "<span style='color: red'>%s</span>" % item['base']
if 'time' in item:
item['log'] = log_name(info['id'], item)
if 'git_commit_human' not in item:
item['git_commit_human'] = "%s new commits" % len(item['log'])
for x in ('commit', 'base', 'merge'):
field = 'git_%s_human' % x
item[field] = format_git_describe(item.get(field, None))
yield item
def normalize_plugin(plugin):
while len(plugin) < 3:
plugin.append(None)
return plugin
def sort_fields(items):
return sorted(items, key=(lambda x: (x[0] != 'title', x)))
return render_template("ticket.html", reports=preprocess_reports(info['reports']), ticket=ticket, info=format_info(info), status=get_ticket_status(info, base=base)[2], normalize_plugin=normalize_plugin, sort_fields=sort_fields)
def test_a_ticket(self, ticket=None):
self.reload_config()
if ticket is None:
ticket = self.get_ticket()
else:
ticket = None, scrape(int(ticket))
if not ticket:
print "No more tickets."
time.sleep(self.config['idle'])
return
rating, ticket = ticket
print "\n" * 2
print "=" * 30, ticket['id'], "=" * 30
print ticket['title']
print "score", rating
print "\n" * 2
self.log_dir = self.sage_root + "/logs/patchbot"
if not os.path.exists(self.log_dir):
os.makedirs(self.log_dir)
log = '%s/%s-log.txt' % (self.log_dir, ticket['id'])
history = open("%s/history.txt" % self.log_dir, "a")
history.write("%s %s\n" % (datetime(), ticket['id']))
history.close()
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=log)
plugins_results = []
try:
t = Timer()
with Tee(log, time=True, timeout=self.config['timeout'], timer=t):
print "Sage Patchbot", patchbot_version.get_version()
if ticket['spkgs']:
state = 'spkg'
print "\n".join(ticket['spkgs'])
print
for spkg in ticket['spkgs']:
print
print '+' * 10, spkg, '+' * 10
print
try:
self.check_spkg(spkg)
except Exception:
traceback.print_exc()
t.finish(spkg)
if not ticket['spkgs']:
state = 'started'
os.environ['MAKE'] = "make -j%s" % self.config['parallelism']
os.environ['SAGE_ROOT'] = self.sage_root
os.environ['GIT_AUTHOR_NAME'] = os.environ['GIT_COMMITTER_NAME'] = 'patchbot'
os.environ['GIT_AUTHOR_EMAIL'] = os.environ['GIT_COMMITTER_EMAIL'] = 'patchbot@localhost'
os.environ['GIT_AUTHOR_DATE'] = os.environ['GIT_COMMITTER_DATE'] = '1970-01-01T00:00:00'
pull_from_trac(self.sage_root, ticket['id'], force=True, use_ccache=self.config['use_ccache'])
t.finish("Apply")
state = 'applied'
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=log, pending_status=state)
do_or_die("$MAKE")
t.finish("Build")
state = 'built'
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=log, pending_status=state)
# TODO: Exclude dependencies.
patch_dir = tempfile.mkdtemp()
if ticket['id'] != 0:
do_or_die("git format-patch -o '%s' patchbot/base..patchbot/ticket_merged" % patch_dir)
kwds = {
"original_dir": self.sage_root,
"patched_dir": os.getcwd(),
"patches": [os.path.join(patch_dir, p) for p in os.listdir(patch_dir)],
"sage_binary": os.path.join(os.getcwd(), 'sage'),
"dry_run": self.dry_run,
}
for name, plugin in self.config['plugins']:
try:
if ticket['id'] != 0 and os.path.exists(os.path.join(self.log_dir, '0', name)):
baseline = pickle.load(open(os.path.join(self.log_dir, '0', name)))
else:
baseline = None
print plugin_boundary(name)
do_or_die("git checkout patchbot/ticket_merged")
res = plugin(ticket, is_git=True, baseline=baseline, **kwds)
passed = True
except Exception:
traceback.print_exc()
passed = False
res = None
finally:
if isinstance(res, PluginResult):
if res.baseline is not None:
plugin_dir = os.path.join(self.log_dir, str(ticket['id']))
if not os.path.exists(plugin_dir):
os.mkdir(plugin_dir)
pickle.dump(res.baseline, open(os.path.join(plugin_dir, name), 'w'))
passed = res.status == PluginResult.Passed
print name, res.status
plugins_results.append((name, passed, res.data))
else:
plugins_results.append((name, passed, None))
t.finish(name)
print plugin_boundary(name, end=True)
plugins_passed = all(passed for (name, passed, data) in plugins_results)
self.report_ticket(ticket, status='Pending', log=log, pending_status='plugins_passed' if plugins_passed else 'plugins_failed')
if self.plugin_only:
state = 'plugins' if plugins_passed else 'plugins_failed'
else:
if self.dry_run:
test_target = "$SAGE_ROOT/src/sage/misc/a*.py"
# TODO: Remove
test_target = "$SAGE_ROOT/src/sage/doctest/*.py"
else:
test_target = "--all --long"
if self.config['parallelism'] > 1:
test_cmd = "-tp %s" % self.config['parallelism']
else:
test_cmd = "-t"
do_or_die("$SAGE_ROOT/sage %s %s" % (test_cmd, test_target))
t.finish("Tests")
state = 'tested'
if not plugins_passed:
state = 'tests_passed_plugins_failed'
except (urllib2.HTTPError, socket.error):
# Don't report failure because the network/trac died...
print
t.print_all()
traceback.print_exc()
state = 'network_error'
except Exception:
traceback.print_exc()
for _ in range(5):
try:
print "Reporting", ticket['id'], status[state]
self.report_ticket(ticket, status=status[state], log=log, plugins=plugins_results, dry_run=self.dry_run)
print "Done reporting", ticket['id']
break
except IOError:
traceback.print_exc()
time.sleep(self.config['idle'])
else:
print "Error reporting", ticket['id']
maybe_temp_root = os.environ['SAGE_ROOT']
if maybe_temp_root.endswith("-sage-git-temp-%s" % ticket['id']):
shutil.rmtree(maybe_temp_root)
return status[state]
def render_ticket(ticket):
"""
reports on a given ticket
possible options: ?force and ?kick
?force will refresh the info in the patchbot-server database
?kick will tell the patchbot-clients to retry the ticket
?base to select reports according to their base
"""
latest = latest_base()
if 'base' in request.args:
chosen_base = request.args.get('base')
else:
chosen_base = 'all' if ticket != 0 else 'develop'
if chosen_base == 'latest' or chosen_base == 'develop':
chosen_base = latest
try:
info = scrape(ticket, db=db, force='force' in request.args)
except:
info = tickets.find_one({'id': ticket})
if info is None:
return "No such ticket."
if 'kick' in request.args:
info['retry'] = True
db.save_ticket(info)
if 'reports' in info:
info['reports'].sort(lambda a, b: -cmp(a['time'], b['time']))
else:
info['reports'] = []
base_reports = base_reports_by_machine_and_base()
old_reports = list(info['reports'])
prune_pending(info)
if old_reports != info['reports']:
db.save_ticket(info)
def format_info(info):
new_info = {}
for key, value in info.items():
if key in ['patches', 'reports', 'pending']:
pass
elif key == 'depends_on':
deps_status = {}
def is_int(a):
try:
int(a)
return True
except ValueError:
return False
for dep in tickets.find({'id': {'$in': [int(a) for a in value
if is_int(a)]}},
['status', 'id']):
if 'closed' in dep['status']:
dep['style'] = 'text-decoration: line-through'
else:
dep['style'] = ''
deps_status[dep['id']] = dep
new_info[key] = ', '.join("<img src='/ticket/%s/status.svg?fast' height=16><a href='/ticket/%s' style='%s'>%s</a>" % (a, a, deps_status[a]['style'], a) for a in value)
elif key == 'authors':
new_info[key] = ', '.join("<a href='/ticket/?author=%s'>%s</a>" % (a, a) for a in value)
elif key == 'authors_fullnames':
link = u"<a href='http://git.sagemath.org/sage.git/log/?qt=author&q={}'>{}</a>"
auths = u", ".join(link.format(a.replace(u" ", u"%20"), a)
for a in value)
trust_check = u"(<a href='/trust_check?who="
trust_check += u",".join(u"{}".format(a) for a in value)
trust_check += u"'>Check trust</a>) "
new_info[key] = trust_check + auths
elif key == 'participants':
parts = ', '.join("<a href='/ticket/?participant=%s'>%s</a>" % (a, a) for a in value)
new_info[key] = parts
elif key == 'git_branch':
new_info[key] = '<a href="http://git.sagemath.org/sage.git/log/?h=%s">%s</a>' % (value, value)
elif key == 'spkgs':
new_info[key] = ', '.join("<a href='%s'>%s</a>" % (a, a) for a in value)
elif isinstance(value, list):
new_info[key] = ', '.join(value)
elif key not in ('id', '_id'):
new_info[key] = value
return new_info
def format_git_describe(res):
if res:
if '-' in res:
tag, commits = res.split('-')[:2]
return "%s + %s commits" % (tag, commits)
elif 'commits' in res:
# old style
return res
else:
return res + " + 0 commits"
else:
return '?'
def preprocess_reports(all):
for item in all:
base_of_this_report = item['base']
base_report = base_reports.get(item['base'] + "/" + "/".join(item['machine']), base_reports.get(item['base']))
if base_report:
item['base_log'] = quote(log_name(0, base_report))
if 'git_base' in item:
git_log = item.get('git_log')
item['git_log_len'] = '?' if git_log is None else len(git_log)
item['raw_base'] = item['base']
if compare_version(item['base'], latest) < 0:
item['base'] = "<span style='color: red'>%s</span>" % item['base']
if 'time' in item:
item['log'] = log_name(info['id'], item)
if 'git_commit_human' not in item:
item['git_commit_human'] = "%s new commits" % len(item['log'])
for x in ('commit', 'base', 'merge'):
field = 'git_%s_human' % x
item[field] = format_git_describe(item.get(field, None))
if chosen_base == 'all' or chosen_base == base_of_this_report:
yield item
def normalize_plugin(plugin):
while len(plugin) < 3:
plugin.append(None)
return plugin
def sort_fields(items):
return sorted(items, key=(lambda x: (x[0] != 'title', x)))
status_data = get_ticket_status(info, base=latest)[1] # single status
return render_template("ticket.html",
reports=preprocess_reports(info['reports']),
ticket=ticket, info=format_info(info),
status=status_data,
normalize_plugin=normalize_plugin,
sort_fields=sort_fields)
def test_a_ticket(self, ticket=None):
self.reload_config()
if ticket is None:
ticket = self.get_ticket()
else:
ticket = None, scrape(int(ticket))
if not ticket:
print "No more tickets."
time.sleep(conf['idle'])
return
rating, ticket = ticket
print "\n" * 2
print "=" * 30, ticket['id'], "=" * 30
print ticket['title']
print "score", rating
print "\n" * 2
log_dir = self.sage_root + "/logs"
if not os.path.exists(log_dir):
os.mkdir(log_dir)
log = '%s/%s-log.txt' % (log_dir, ticket['id'])
if not self.plugin_only:
self.report_ticket(ticket, status='Pending', log=None)
plugins_results = []
try:
with Tee(log, time=True, timeout=self.config['timeout']):
t = Timer()
start_time = time.time()
if ticket['spkgs']:
state = 'spkg'
print "\n".join(ticket['spkgs'])
print
for spkg in ticket['spkgs']:
print
print '+' * 10, spkg, '+' * 10
print
try:
self.check_spkg(spkg)
except Exception:
traceback.print_exc()
t.finish(spkg)
else:
state = 'started'
os.environ['MAKE'] = "make -j%s" % self.config['parallelism']
os.environ['SAGE_ROOT'] = self.sage_root
# TODO: Ensure that sage-main is pristine.
pull_from_trac(self.sage_root, ticket['id'], force=True)
t.finish("Apply")
state = 'applied'
do_or_die('$SAGE_ROOT/sage -b %s' % ticket['id'])
t.finish("Build")
state = 'built'
working_dir = "%s/devel/sage-%s" % (self.sage_root, ticket['id'])
# Only the ones on this ticket.
patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):]
kwds = {
"original_dir": "%s/devel/sage-0" % self.sage_root,
"patched_dir": working_dir,
"patches": ["%s/devel/sage-%s/.hg/patches/%s" % (self.sage_root, ticket['id'], p) for p in patches if p],
"sage_binary": os.path.join(self.sage_root, 'sage')
}
for name, plugin in self.config['plugins']:
try:
if ticket['id'] != 0 and os.path.exists(os.path.join(log_dir, '0', name)):
baseline = pickle.load(open(os.path.join(log_dir, '0', name)))
else:
baseline = None
print plugin_boundary(name)
res = plugin(ticket, baseline=baseline, **kwds)
passed = True
except Exception:
traceback.print_exc()
passed = False
res = None
finally:
if isinstance(res, PluginResult):
if res.baseline is not None:
plugin_dir = os.path.join(log_dir, str(ticket['id']))
if not os.path.exists(plugin_dir):
os.mkdir(plugin_dir)
pickle.dump(res.baseline, open(os.path.join(plugin_dir, name), 'w'))
passed = res.status == PluginResult.Passed
print name, res.status
plugins_results.append((name, passed, res.data))
else:
plugins_results.append((name, passed, None))
t.finish(name)
print plugin_boundary(name, end=True)
plugins_passed = all(passed for (name, passed, data) in plugins_results)
if self.plugin_only:
state = 'plugins' if plugins_passed else 'plugins_failed'
else:
if self.dry_run:
test_dirs = ["$SAGE_ROOT/devel/sage-%s/sage/misc/a*.py" % (ticket['id'])]
else:
test_dirs = ["--sagenb"] + ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs]
if conf['parallelism'] > 1:
test_cmd = "-tp %s" % conf['parallelism']
else:
test_cmd = "-t"
do_or_die("$SAGE_ROOT/sage %s %s" % (test_cmd, ' '.join(test_dirs)))
t.finish("Tests")
state = 'tested'
if not plugins_passed:
state = 'tests_passed_plugins_failed'
print
t.print_all()
except urllib2.HTTPError:
# Don't report failure because the network/trac died...
traceback.print_exc()
return 'Pending'
except Exception:
traceback.print_exc()
for _ in range(5):
try:
print "Reporting", ticket['id'], status[state]
if not self.dry_run:
self.report_ticket(ticket, status=status[state], log=log, plugins=plugins_results)
print "Done reporting", ticket['id']
break
except urllib2.HTTPError:
traceback.print_exc()
time.sleep(conf['idle'])
else:
print "Error reporting", ticket['id']
if not conf['keep_open_branches'] and str(ticket['id']) != '0' and not ticket['spkgs']:
shutil.rmtree(os.path.join(self.sage_root, "devel", "sage-%s" %ticket['id']))
return status[state]