def deserialize(buildobj, args):
if buildobj['type'] == 'compile':
# See https://github.com/mozilla/areweslimyet/issues/47
raise Exception("Build type 'compile' is not currently supported")
elif buildobj['type'] == 'tinderbox':
build = BuildGetter.TinderboxBuild(buildobj['timestamp'],
buildobj['branch'])
elif buildobj['type'] == 'nightly':
build = BuildGetter.NightlyBuild(
parse_nightly_time(buildobj['for']))
elif buildobj['type'] == 'ftp':
build = BuildGetter.FTPBuild(buildobj['path'])
elif buildobj['type'] == 'try':
build = BuildGetter.TryBuild(buildobj['changeset'])
else:
raise Exception("Unkown build type %s" % buildobj['type'])
ret = BatchBuild(build, buildobj['revision'])
ret.series = buildobj['series']
ret.uid = buildobj['uid']
ret.timestamp = buildobj['timestamp']
ret.note = buildobj['note']
ret.started = buildobj['started']
ret.finished = buildobj['finished']
ret.force = buildobj['force']
return ret
def deserialize(buildobj, args):
if buildobj['type'] == 'compile':
if args.get('logdir'):
logfile = os.path.join(args.get('logdir'), "%s.build.log" % (buildobj['revision'],))
else:
logfile = None
build = BuildGetter.CompileBuild(args.get('repo'), args.get('mozconfig'), args.get('objdir'), pull=True, commit=buildobj['revision'], log=logfile)
elif buildobj['type'] == 'tinderbox':
build = BuildGetter.TinderboxBuild(buildobj['timestamp'], buildobj['branch'])
elif buildobj['type'] == 'nightly':
build = BuildGetter.NightlyBuild(parse_nightly_time(buildobj['for']))
elif buildobj['type'] == 'ftp':
build = BuildGetter.FTPBuild(buildobj['path'])
else:
raise Exception("Unkown build type %s" % buildobj['type'])
ret = BatchBuild(build, buildobj['revision'])
ret.series = buildobj['series']
ret.uid = buildobj['uid']
ret.timestamp = buildobj['timestamp']
ret.note = buildobj['note']
ret.started = buildobj['started']
ret.finished = buildobj['finished']
ret.force = buildobj['force']
return ret
def _process_batch_inner(globalargs, batchargs, hook):
if not batchargs['firstbuild']:
raise Exception("--firstbuild is required")
mode = batchargs['mode']
dorange = 'lastbuild' in batchargs and batchargs['lastbuild']
builds = []
# Queue builds
if mode == 'nightly':
startdate = parse_nightly_time(batchargs['firstbuild'])
if dorange:
enddate = parse_nightly_time(batchargs['lastbuild'])
dates = range(startdate.toordinal(), enddate.toordinal() + 1)
else:
dates = [ startdate.toordinal() ]
for x in dates:
builds.append(BuildGetter.NightlyBuild(datetime.date.fromordinal(x)))
elif mode == 'tinderbox':
startdate = float(batchargs['firstbuild'])
if dorange:
enddate = float(batchargs['lastbuild'])
tinderbuilds = BuildGetter.list_tinderbox_builds(startdate, enddate)
for x in tinderbuilds:
builds.append(BuildGetter.TinderboxBuild(x))
else:
builds.append(BuildGetter.TinderboxBuild(startdate))
elif mode == 'ftp':
path = batchargs['firstbuild']
builds.append(BuildGetter.FTPBuild(path))
elif mode == 'try':
path = batchargs['firstbuild']
builds.append(BuildGetter.TryBuild(path))
elif mode == 'compile':
# See https://github.com/mozilla/areweslimyet/issues/47
raise Exception("Build type 'compile' is not currently supported")
else:
raise Exception("Unknown mode %s" % mode)
readybuilds = []
skippedbuilds = []
force = batchargs.get('force') if batchargs.get('force') else globalargs.get('force')
for build in builds:
rev = build.get_revision()
#HACKITY HACK HACK HACK
build._scraper = None
build = BatchBuild(build, rev)
build.force = force
build.series = batchargs.get('series')
if not build.build.get_valid():
# Can happen with FTP builds we failed to lookup on ftp.m.o, or any
# builds that arn't found in pushlog
build.note = "Build is not found or missing from pushlog"
elif hook and not hook.should_test(build, globalargs):
if not build.note:
build.note = "Build skipped by tester";
else:
readybuilds.append(build)
continue
build.finished = time.time()
skippedbuilds.append(build)
return [ readybuilds, skippedbuilds ]
def _process_batch_inner(globalargs, batchargs, hook):
if not batchargs['firstbuild']:
raise Exception("--firstbuild is required")
if not globalargs.get('no_pull'):
# Do a tip lookup to pull the repo so get_full_revision is up to date
BuildGetter.get_hg_range(globalargs.get('repo'), '.', '.', True)
mode = batchargs['mode']
dorange = 'lastbuild' in batchargs and batchargs['lastbuild']
builds = []
# Queue builds
if mode == 'nightly':
startdate = parse_nightly_time(batchargs['firstbuild'])
if dorange:
enddate = parse_nightly_time(batchargs['lastbuild'])
dates = range(startdate.toordinal(), enddate.toordinal() + 1)
else:
dates = [ startdate.toordinal() ]
for x in dates:
builds.append(BuildGetter.NightlyBuild(datetime.date.fromordinal(x)))
elif mode == 'tinderbox':
startdate = float(batchargs['firstbuild'])
if dorange:
enddate = float(batchargs['lastbuild'])
tinderbuilds = BuildGetter.list_tinderbox_builds(startdate, enddate)
for x in tinderbuilds:
builds.append(BuildGetter.TinderboxBuild(x))
else:
builds.append(BuildGetter.TinderboxBuild(startdate))
elif mode == 'ftp':
builds.append(BuildGetter.FTPBuild(batchargs['firstbuild']))
elif mode == 'compile':
repo = batchargs.get('repo') if batchargs.get('repo') else globalargs.get('repo')
objdir = batchargs.get('objdir') if batchargs.get('objdir') else globalargs.get('objdir')
mozconfig = batchargs.get('mozconfig') if batchargs.get('mozconfig') else globalargs.get('mozconfig')
if not repo or not mozconfig or not objdir:
raise Exception("Build mode requires --repo, --mozconfig, and --objdir to be set")
if dorange:
lastbuild = batchargs['lastbuild']
else:
lastbuild = batchargs['firstbuild']
for commit in BuildGetter.get_hg_range(repo, batchargs['firstbuild'], lastbuild, not globalargs.get("no_pull")):
if globalargs.get('logdir'):
logfile = os.path.join(globalargs.get('logdir'), "%s.build.log" % (commit,))
else:
logfile = None
builds.append(BuildGetter.CompileBuild(repo, mozconfig, objdir, pull=False, commit=commit, log=logfile))
else:
raise Exception("Unknown mode %s" % mode)
readybuilds = []
skippedbuilds = []
force = batchargs.get('force') if batchargs.get('force') else globalargs.get('force')
for build in builds:
rev = build.get_revision()
build = BatchBuild(build, rev)
build.force = force
build.series = batchargs.get('series')
if not build.build.get_valid():
# Can happen with FTP builds we failed to lookup on ftp.m.o, or any
# builds that arn't found in pushlog
build.note = "Build is not found or missing from pushlog"
elif hook and not hook.should_test(build, globalargs):
if not build.note:
build.note = "Build skipped by tester";
else:
readybuilds.append(build)
continue
build.finished = time.time()
skippedbuilds.append(build)
return [ readybuilds, skippedbuilds ]
err("Incorrect usage. See comments")
batchdir = sys.argv[1]
knownbuilds = sys.argv[2]
branch = sys.argv[3]
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "..", "benchtester")))
import BuildGetter
BuildGetter.output = sys.stderr
knownfp = file(knownbuilds, "r")
old_builds = json.load(knownfp)
knownfp.close()
builds = BuildGetter.list_tinderbox_builds(branch=branch)
if not len(builds):
stat("No builds to queue")
sys.exit(0)
stat("Comparing %u builds on tinderbox with %u known builds" % (len(builds), len(old_builds)))
def queue_build(timestamp):
out = {"mode": "tinderbox", "firstbuild": str(timestamp), "note": "Auto-queued by cron"}
batchfile = os.path.join(batchdir, "tinderbox-%u.autoqueue" % (timestamp,))
if os.path.exists(batchfile):
err('Failed: file "%s" already exists' % (batchfile,))
f = open(batchfile, "w")
json.dump(out, f)
sys.path.append(os.path.abspath('benchtester'))
import BuildGetter
if len(sys.argv) != 3:
sys.stderr.write("Requires arguments: sqlitedb branchname")
sys.exit(1)
gSqlite = sys.argv[1]
gBranch = sys.argv[2] # e.g. integration/mozilla-inbound
sql = sqlite3.connect(gSqlite, timeout=900)
cur = sql.cursor()
cur.execute("SELECT `id`, `name`, `time` FROM benchtester_builds")
builds = 0
touched = 0
for build in cur.fetchall():
builds += 1
newstamp = BuildGetter.pushlog_lookup(build[1])[1]
if not newstamp:
print("!! Couldn't lookup build %s" % (build[1],))
if int(newstamp) != int(build[2]):
touched += 1
print("Updating build %s from %u to %u" % (build[1], build[2], newstamp))
cur.execute("UPDATE `benchtester_builds` SET `time` = ? WHERE `id` = ?",
[newstamp, build[0]])
sql.commit()
print("Looked at %u builds and updated %u" % (builds, touched))
batchdir = sys.argv[1]
knownbuilds = sys.argv[2]
branch = sys.argv[3]
sys.path.append(
os.path.abspath(
os.path.join(os.path.dirname(sys.argv[0]), "..", "benchtester")))
import BuildGetter
BuildGetter.output = sys.stderr
knownfp = file(knownbuilds, 'r')
old_builds = json.load(knownfp)
knownfp.close()
builds = BuildGetter.list_tinderbox_builds(branch=branch)
if not len(builds):
stat("No builds to queue")
sys.exit(0)
stat("Comparing %u builds on tinderbox with %u known builds" %
(len(builds), len(old_builds)))
def queue_build(timestamp):
out = {
'mode': 'tinderbox',
'firstbuild': str(timestamp),
'note': 'Auto-queued by cron'
}
batchfile = os.path.join(batchdir,
# Run from root
sys.path.append(os.path.abspath("benchtester"))
import BuildGetter
if len(sys.argv) != 3:
sys.stderr.write("Requires arguments: sqlitedb branchname")
sys.exit(1)
gSqlite = sys.argv[1]
gBranch = sys.argv[2] # e.g. integration/mozilla-inbound
sql = sqlite3.connect(gSqlite, timeout=900)
cur = sql.cursor()
cur.execute("SELECT `id`, `name`, `time` FROM benchtester_builds")
builds = 0
touched = 0
for build in cur.fetchall():
builds += 1
newstamp = BuildGetter.pushlog_lookup(build[1])[1]
if not newstamp:
print("!! Couldn't lookup build %s" % (build[1],))
if int(newstamp) != int(build[2]):
touched += 1
print("Updating build %s from %u to %u" % (build[1], build[2], newstamp))
cur.execute("UPDATE `benchtester_builds` SET `time` = ? WHERE `id` = ?", [newstamp, build[0]])
sql.commit()
print("Looked at %u builds and updated %u" % (builds, touched))
def _process_batch_inner(globalargs, batchargs, hook):
if not batchargs['firstbuild']:
raise Exception("--firstbuild is required")
mode = batchargs['mode']
dorange = 'lastbuild' in batchargs and batchargs['lastbuild']
builds = []
# Queue builds
if mode == 'nightly':
startdate = parse_nightly_time(batchargs['firstbuild'])
if dorange:
enddate = parse_nightly_time(batchargs['lastbuild'])
dates = range(startdate.toordinal(), enddate.toordinal() + 1)
else:
dates = [startdate.toordinal()]
for x in dates:
builds.append(
BuildGetter.NightlyBuild(datetime.date.fromordinal(x)))
elif mode == 'tinderbox':
startdate = float(batchargs['firstbuild'])
if dorange:
enddate = float(batchargs['lastbuild'])
tinderbuilds = BuildGetter.list_tinderbox_builds(
startdate, enddate)
for x in tinderbuilds:
builds.append(BuildGetter.TinderboxBuild(x))
else:
builds.append(BuildGetter.TinderboxBuild(startdate))
elif mode == 'ftp':
path = batchargs['firstbuild']
builds.append(BuildGetter.FTPBuild(path))
elif mode == 'try':
path = batchargs['firstbuild']
builds.append(BuildGetter.TryBuild(path))
elif mode == 'compile':
# See https://github.com/mozilla/areweslimyet/issues/47
raise Exception("Build type 'compile' is not currently supported")
else:
raise Exception("Unknown mode %s" % mode)
readybuilds = []
skippedbuilds = []
force = batchargs.get('force') if batchargs.get(
'force') else globalargs.get('force')
for build in builds:
rev = build.get_revision()
#HACKITY HACK HACK HACK
build._scraper = None
build = BatchBuild(build, rev)
build.force = force
build.series = batchargs.get('series')
if not build.build.get_valid():
# Can happen with FTP builds we failed to lookup on ftp.m.o, or any
# builds that arn't found in pushlog
build.note = "Build is not found or missing from pushlog"
elif hook and not hook.should_test(build, globalargs):
if not build.note:
build.note = "Build skipped by tester"
else:
readybuilds.append(build)
continue
build.finished = time.time()
skippedbuilds.append(build)
return [readybuilds, skippedbuilds]
