def __init__(self, sage_root, server, config_path, dry_run=False, plugin_only=False):
self.sage_root = sage_root
self.server = server
self.base = get_base(sage_root)
self.dry_run = dry_run
self.plugin_only = plugin_only
self.config_path = config_path
self.reload_config()
def test_code():
# Read in adjusted closing prices for given symbols, date range
orders = read_csv('orders.csv')
dates, symbols = get_base(orders)
prices_all = get_data(symbols, dates) # automatically adds SPY
prices = prices_all[symbols] # only portfolio symbols
prices_SPY = prices_all['SPY'] # only SPY, for comparison later
# Create a dataframe which has all values as zero with index as dates and columns as symbols.
trade_matrix = pd.DataFrame(0, index=dates, columns=symbols)
print trade_matrix
def main(args):
global conf
# Most configuration is done in the config file, which is reread between
# each ticket for live configuration of the patchbot.
parser = OptionParser()
parser.add_option("--config", dest="config")
parser.add_option("--sage-root", dest="sage_root", default=os.environ.get('SAGE_ROOT'))
parser.add_option("--server", dest="server", default="http://patchbot.sagemath.org/")
parser.add_option("--count", dest="count", default=1000000)
parser.add_option("--ticket", dest="ticket", default=None)
parser.add_option("--list", dest="list", default=False)
parser.add_option("--skip-base", dest="skip_base", default=False)
(options, args) = parser.parse_args(args)
conf_path = options.config and os.path.abspath(options.config)
if options.ticket:
tickets = [int(t) for t in options.ticket.split(',')]
count = len(tickets)
else:
tickets = None
count = int(options.count)
conf = get_conf(conf_path, options.server)
if options.list:
for score, ticket in get_ticket(base=get_base(options.sage_root), server=options.server, return_all=True, **conf):
print score, ticket['id'], ticket['title']
print ticket
print
sys.exit(0)
print "WARNING: Assuming sage-main is pristine."
if options.sage_root == os.environ.get('SAGE_ROOT'):
print "WARNING: Do not use this copy of sage while the patchbot is running."
if not options.skip_base:
clean = lookup_ticket(options.server, 0)
def good(report):
return report['machine'] == conf['machine'] and report['status'] == 'TestsPassed'
if not any(good(report) for report in current_reports(clean, base=get_base(options.sage_root))):
res = test_a_ticket(ticket=0, sage_root=options.sage_root, server=options.server)
if res != 'TestsPassed':
print "\n\n"
while True:
print "Failing tests in your install: %s. Continue anyways? [y/N] " % res
ans = sys.stdin.readline().lower().strip()
if ans == '' or ans[0] == 'n':
sys.exit(1)
elif ans[0] == 'y':
break
for _ in range(count):
try:
if tickets:
ticket = tickets.pop(0)
else:
ticket = None
conf = get_conf(conf_path, options.server)
if check_time_of_day(conf['time_of_day']):
test_a_ticket(ticket=ticket, sage_root=options.sage_root, server=options.server)
else:
print "Idle."
time.sleep(conf['idle'])
except urllib2.HTTPError:
traceback.print_exc()
time.sleep(conf['idle'])
def test_a_ticket(sage_root, server, ticket=None, nodocs=False):
base = get_base(sage_root)
if ticket is None:
ticket = get_ticket(base=base, server=server, **conf)
else:
ticket = None, scrape(int(ticket))
if not ticket:
print "No more tickets."
if random.random() < 0.01:
cleanup(sage_root, server)
time.sleep(conf['idle'])
return
rating, ticket = ticket
print "\n" * 2
print "=" * 30, ticket['id'], "=" * 30
print ticket['title']
print "score", rating
print "\n" * 2
log_dir = sage_root + "/logs"
if not os.path.exists(log_dir):
os.mkdir(log_dir)
log = '%s/%s-log.txt' % (log_dir, ticket['id'])
report_ticket(server, ticket, status='Pending', base=base, machine=conf['machine'], user=conf['user'], log=None)
plugins_results = []
try:
with Tee(log, time=True, timeout=conf['timeout']):
t = Timer()
start_time = time.time()
state = 'started'
os.environ['MAKE'] = "make -j%s" % conf['parallelism']
os.environ['SAGE_ROOT'] = sage_root
# TODO: Ensure that sage-main is pristine.
pull_from_trac(sage_root, ticket['id'], force=True)
t.finish("Apply")
state = 'applied'
do_or_die('$SAGE_ROOT/sage -b %s' % ticket['id'])
t.finish("Build")
state = 'built'
working_dir = "%s/devel/sage-%s" % (sage_root, ticket['id'])
# Only the ones on this ticket.
patches = os.popen2('hg --cwd %s qapplied' % working_dir)[1].read().strip().split('\n')[-len(ticket['patches']):]
kwds = {
"original_dir": "%s/devel/sage-0" % sage_root,
"patched_dir": working_dir,
"patches": ["%s/devel/sage-%s/.hg/patches/%s" % (sage_root, ticket['id'], p) for p in patches if p],
}
for name, plugin in conf['plugins']:
try:
print plugin_boundary(name)
plugin(ticket, **kwds)
passed = True
except Exception:
traceback.print_exc()
passed = False
finally:
t.finish(name)
print plugin_boundary(name, end=True)
plugins_results.append((name, passed))
test_dirs = ["$SAGE_ROOT/devel/sage-%s/%s" % (ticket['id'], dir) for dir in all_test_dirs]
if conf['parallelism'] > 1:
test_cmd = "-tp %s" % conf['parallelism']
else:
test_cmd = "-t"
do_or_die("$SAGE_ROOT/sage %s -sagenb %s" % (test_cmd, ' '.join(test_dirs)))
#do_or_die("$SAGE_ROOT/sage -t $SAGE_ROOT/devel/sage-%s/sage/rings/integer.pyx" % ticket['id'])
#do_or_die('sage -testall')
t.finish("Tests")
state = 'tested'
if not all(passed for name, passed in plugins_results):
state = 'failed_plugin'
print
t.print_all()
except Exception:
traceback.print_exc()
for _ in range(5):
try:
print "Reporting", ticket['id'], status[state]
report_ticket(server, ticket, status=status[state], base=base, machine=conf['machine'], user=conf['user'], log=log, plugins=plugins_results)
print "Done reporting", ticket['id']
break
except urllib2.HTTPError:
traceback.print_exc()
time.sleep(conf['idle'])
else:
print "Error reporting", ticket['id']
return status[state]
def pull_from_trac(sage_root, ticket, branch=None, force=None, interactive=None):
# Should we set/unset SAGE_ROOT and SAGE_BRANCH here? Fork first?
if branch is None:
branch = str(ticket)
if not os.path.exists('%s/devel/sage-%s' % (sage_root, branch)):
do_or_die('%s/sage -b main' % (sage_root,))
do_or_die('%s/sage -clone %s' % (sage_root, branch))
os.chdir('%s/devel/sage-%s' % (sage_root, branch))
if interactive:
raise NotImplementedError
if not os.path.exists('.hg/patches'):
do_or_die('hg qinit')
series = []
elif not os.path.exists('.hg/patches/series'):
series = []
else:
series = open('.hg/patches/series').read().split('\n')
base = get_base(sage_root)
desired_series = []
seen_deps = []
def append_patch_list(ticket, dependency=False):
if ticket in seen_deps:
return
print "Looking at #%s" % ticket
seen_deps.append(ticket)
data = scrape(ticket)
if dependency and 'closed' in data['status']:
merged = data.get('merged')
if merged is None:
merged = data.get('milestone')
if merged is None or compare_version(merged, base) <= 0:
print "#%s already applied (%s <= %s)" % (ticket, merged, base)
return
if data['spkgs']:
raise NotImplementedError, "Spkgs not yet handled."
if data['depends_on']:
for dep in data['depends_on']:
if isinstance(dep, basestring) and '.' in dep:
if compare_version(base, dep) < 0:
raise ValueError, "%s < %s for %s" % (base, dep, ticket)
continue
append_patch_list(dep, dependency=True)
print "Patches for #%s:" % ticket
print " " + "\n ".join(data['patches'])
for patch in data['patches']:
patchfile, hash = patch.split('#')
desired_series.append((hash, patchfile, get_patch_url(ticket, patchfile)))
append_patch_list(ticket)
ensure_safe(series)
ensure_safe(patch for hash, patch, url in desired_series)
last_good_patch = '-a'
to_push = list(desired_series)
for series_patch, (hash, patch, url) in zip(series, desired_series):
if not series_patch:
break
next_hash = digest(open('.hg/patches/%s' % series_patch).read())
# print next_hash, hash, series_patch
if next_hash == hash:
to_push.pop(0)
last_good_patch = series_patch
else:
break
try:
if last_good_patch != '-a':
# In case it's not yet pushed...
if last_good_patch not in os.popen2('hg qapplied')[1].read().split('\n'):
do_or_die('hg qpush %s' % last_good_patch)
do_or_die('hg qpop %s' % last_good_patch)
for hash, patch, url in to_push:
if patch in series:
if not force:
raise Exception, "Duplicate patch: %s" % patch
old_patch = patch
while old_patch in series:
old_patch += '-old'
do_or_die('hg qrename %s %s' % (patch, old_patch))
try:
do_or_die('hg qimport %s' % url)
except Exception, exn:
time.sleep(30)
try:
do_or_die('hg qimport %s' % url)
except Exception, exn:
raise urllib2.HTTPError(exn)
do_or_die('hg qpush')
