def get_bug(self, bug, attachments=True, comments=True, history=True):
"""Fetch Bug ``bug``."""
tmp = {'attachmentdata': attachments, 'comments': comments,
'history': history}
params = dict((k, int(v)) for k, v in tmp.items())
url = urljoin(API_ROOT, 'bug/%s?%s' % (bug, self.qs(**params)))
return Bug.get(url)
def _attach(self, bug_id, filename, description, is_patch=False,
reviewer=None, content_type='text/plain'):
"""Create a new attachment."""
fields = {'data': base64.b64encode(open(filename).read()),
'encoding': 'base64',
'file_name': filename,
'content_type': content_type,
'description': description,
'is_patch': is_patch,
}
if reviewer is not None:
fields['flags'] = [Flag(type_id=REVIEW, status='?',
requestee=User(name=reviewer))]
url = urljoin(API_ROOT, 'bug/%s/attachment?%s' % (bug_id, self.qs()))
return Attachment(**fields).post_to(url)
def _comment(self, bug_id, comment):
"""Create a new comment."""
url = urljoin(self.API_ROOT, 'bug/%s/comment?%s' % (bug_id, self.qs()))
return Comment(text=comment).post_to(url)
def get_bug_list(self, params={}):
url = url = urljoin(self.API_ROOT, 'bug/?%s' % (self.qs(**params)))
return BugSearch.get(url, http=self.http).bugs
def get_bug(self, bug, include_fields='_default,token,cc,keywords,whiteboard', exclude_fields=None, params={}):
params['include_fields'] = include_fields
params['exclude_fields'] = exclude_fields
url = urljoin(self.API_ROOT, 'bug/%s?%s' % (bug, self.qs(**params)))
return Bug.get(url, http=self.http)
def reproduceBug(self, bug, tipBranch=None, tipBranchRev=None):
bugnum = str(bug.id)
# Determine comment to look at and revision
testCommentIdx = 0
rev = None
if (tipBranch != None and tipBranchRev != None):
rev = tipBranchRev
if (bug.whiteboard != None):
ret = re.compile('\[jsbugmon:([^\]]+)\]').search(bug.whiteboard)
if (ret != None and ret.groups > 1):
wbOpts = ret.group(1).split(",")
for opt in wbOpts:
if (opt.find("=") > 0):
(cmd,param) = opt.split('=')
if (cmd != None and param != None):
if (cmd == "origRev" and rev == None):
rev = param
elif (cmd == "testComment" and param.isdigit()):
testCommentIdx = int(param)
# Look for the first comment
comment = bug.comments[testCommentIdx] if len(bug.comments) > testCommentIdx else None
if (comment == None):
raise BugException("Error: Specified bug does not have any comments")
text = comment.text
# Isolate revision to test for
if (rev == None):
rev = self.extractRevision(text)
else:
# Sanity check of the revision
rev = self.extractRevision(rev)
if (rev == None):
raise BugException("Error: Failed to isolate original revision for test")
buildFlags = []
checkFlags = ["--enable-more-deterministic", "--enable-simulator=arm", "--enable-arm-simulator", "--enable-debug", "--disable-debug", "--enable-optimize", "--disable-optimize"]
for flag in checkFlags:
if (re.search(flag + "[^-a-zA-Z0-9]", text) != None):
buildFlags.append(flag)
viableOptsList = []
opts = []
for opt in self.allowedOpts:
if (text.find(opt) != -1):
opts.append(opt)
viableOptsList.append(opts)
print "Extracted options: %s" % (' '.join(opts))
# Special hack for flags that changed
if "--ion-parallel-compile=off" in opts:
optsCopy = []
for opt in opts:
if opt == "--ion-parallel-compile=off":
optsCopy.append("--ion-offthread-compile=off")
else:
optsCopy.append(opt)
viableOptsList.append(optsCopy)
if (bug.version == "Trunk"):
reponame = "mozilla-central"
elif (bug.version == "Other Branch"):
reponame = "ionmonkey"
else:
raise BugException("Error: Unsupported branch \"" + bug.version + "\" required by bug")
# Default to using the bug.version field as repository specifier
repoDir = os.path.join(self.repoBase, reponame)
# If told to use a different tipBranch, use that for tip testing
if (tipBranch == None):
tipBranch = reponame
tipRepoDir = os.path.join(self.repoBase, tipBranch)
# If we are given a specific revision even for testing, then use
# the tipBranch for all testing, including initial reproduction
if (tipBranchRev != None):
repoDir = tipRepoDir
print "Using repository at %s with revision %s for initial reproduction" % (repoDir, rev)
print "Using repository at %s with tip revision for testing" % (tipRepoDir)
arch = None
archList = None
if (bug.platform == "x86_64"):
arch = "64"
elif (bug.platform == "x86"):
arch = "32"
elif (bug.platform == "All"):
arch = "64"
archList = [ "64", "32" ] # TODO: Detect native platform here
# When auto-detecting, avoid using ARM simulator for now
if "--enable-simulator=arm" in buildFlags:
buildFlags.remove("--enable-simulator=arm")
elif (bug.platform == "ARM"):
arch = "32"
buildFlags.append("--enable-simulator=arm")
else:
raise BugException("Error: Unsupported architecture \"" + bug.platform + "\" required by bug")
# We need at least some shell to extract the test from the bug,
# so we build a debug shell here already
try:
(testShell, testRev) = self.getShell("cache/", arch, "dbg", 0, rev, False, repoDir, buildFlags)
except Exception:
trace = sys.exc_info()[2]
raise InternalException("Failed to compile tip shell (toolchain broken?)"), None, trace
# If the file already exists, then we can reuse it
if testCommentIdx > 0:
testFile = "bug" + str(bugnum) + "-" + str(testCommentIdx) + ".js"
else:
testFile = "bug" + str(bugnum) + ".js"
if (os.path.exists(testFile)):
print "Using existing (cached) testfile " + testFile
else:
# We need to detect where our test is.
blocks = text.split("\n\n")
found = False
cnt = 0
for i,block in enumerate(blocks):
# Write our test to file
outFile = open(testFile, "w")
outFile.write(block)
outFile.close()
print "Testing syntax with shell %s" % testShell
(err, ret) = testBinary(testShell, testFile, [], 0, timeout=30)
if (err.find("SyntaxError") < 0):
# We have found the test (or maybe only the start of the test)
# Try adding more code until we hit an error or are out of
# blocks.
oldBlock = block
curBlock = block
for j,block in enumerate(blocks):
if j > i:
curBlock = curBlock + "\n" + block
# Write our test to file
outFile = open(testFile, "w")
outFile.write(curBlock)
outFile.close()
(err, ret) = testBinary(testShell, testFile, [], 0, timeout=30)
if (err.find("SyntaxError") >= 0):
# Too much, write oldBlock and break
outFile = open(testFile, "w")
outFile.write(oldBlock)
outFile.close()
break
else:
oldBlock = curBlock
found = True
print "Isolated possible testcase starting in textblock " + str(cnt)
break
cnt += 1
if not found:
# First try to find a suitable attachment
attachments = [a for a in bug.attachments if not bool(int(a.is_obsolete))]
for attachment in attachments:
# Seriously, we don't need anything larger than 512kb here^^
if (attachment.size <= 512*1024):
# Refetch attachment with content
url = urljoin(self.apiroot, 'attachment/%s/?%s&attachmentdata=1' % (attachment.id, self.bz.qs()))
attachment = attachment.get(url)
try:
rawData = base64.b64decode(attachment.data)
# Write our data to file
outFile = open(testFile, "w")
outFile.write(rawData)
outFile.close()
(err, ret) = testBinary(testShell, testFile, [], 0, timeout=30)
if (err.find("SyntaxError") < 0):
# Found something that looks like JS :)
found = True
break
except TypeError:
pass
# If we still haven't found any test, give up here...
if not found:
# Ensure we don't cache the wrong test
os.remove(testFile)
raise BugException("Error: Failed to isolate test from comment")
(oouterr, oret) = (None, None)
(origShell, origRev) = (None, None)
# If we have an exact architecture, we will only test that
if (archList == None):
archList = [ arch ]
for compileType in ['dbg', 'opt']:
for archType in archList:
try:
(origShell, origRev) = self.getShell("cache/", archType, compileType, 0, rev, False, repoDir, buildFlags)
except Exception:
# Unlike compilation failures on tip, we must not ignore compilation failures with the original
# revision, as it could mean that the bug was filed with a broken revision.
raise BugException("Error: Failed to compile specified revision %s (maybe try another?)" % rev)
for opts in viableOptsList:
(oouterr, oret) = testBinary(origShell, testFile, opts , 0, verbose=self.options.verbose, timeout=30)
if (oret < 0):
break
# If we reproduced with one arch, then we don't need to try the others
if (oret < 0):
break
print ""
# If we reproduced with dbg, then we don't need to try opt
if (oret < 0):
break
# Check if we reproduced at all (dbg or opt)
if (oret < 0):
print ""
print "Successfully reproduced bug (exit code " + str(oret) + ") on original revision " + rev + ":"
errl = oouterr.split("\n")
if len(errl) > 2: errl = errl[-2:]
for err in errl:
print err
# Try running on tip now
print "Testing bug on tip..."
# Update to tip and cache result:
updated = False
if not self.tipRev.has_key(tipRepoDir):
# If we don't know the tip revision for this branch, update and get it
self.tipRev[tipRepoDir] = self.hgUpdate(tipRepoDir)
updated = True
try:
(tipShell, tipRev) = self.getShell("cache/", archType, compileType, 0, self.tipRev[tipRepoDir], updated, tipRepoDir, buildFlags)
except Exception:
trace = sys.exc_info()[2]
raise InternalException("Failed to compile tip shell (toolchain broken?)"), None, trace
tipOpts = None
for opts in viableOptsList:
(touterr, tret) = testBinary(tipShell, testFile, opts , 0, verbose=self.options.verbose, timeout=30)
if (tret < 0):
tipOpts = opts
break
if (tret < 0):
if (tret == oret):
if (opts == tipOpts):
print "Result: Bug still reproduces"
return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, compileType, buildFlags, BugMonitorResult.statusCodes.REPRODUCED_TIP)
else:
print "Result: Bug still reproduces, but with different options: " + " ".join(tipOpts) # TODO need another code here in the future
return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, compileType, buildFlags, BugMonitorResult.statusCodes.REPRODUCED_TIP)
else:
# Unlikely but possible, switched signal
print "Result: Bug now reproduces with signal " + str(tret) + " (previously " + str(oret) + ")"
return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, compileType, buildFlags, BugMonitorResult.statusCodes.REPRODUCED_SWITCHED)
else:
print "Result: Bug no longer reproduces"
return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, compileType, buildFlags, BugMonitorResult.statusCodes.REPRODUCED_FIXED)
else:
print "Error: Failed to reproduce bug on original revision"
#return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, compileType, buildFlags, BugMonitorResult.statusCodes.FAILED)
return BugMonitorResult(reponame, rev, None, opts, testFile, archType, compileType, buildFlags, BugMonitorResult.statusCodes.FAILED)
def postComment(self, bugnum, comment): url = urljoin(self.apiroot, 'bug/%s/comment?%s' % (bugnum, self.bz.qs())) return Comment(text=comment).post_to(url)
def get_bugs(self, **kwargs):
url = urljoin(self.API_ROOT, 'bug/?%s' % (self.qs(**kwargs)))
return DashBugSearch.get(url, http=self.http).bugs
def reproduceBug(self, bug, tipBranch=None):
# Fetch the bug
#bug = self.fetchBug(bugnum)
bugnum = str(bug.id)
# Determine comment to look at and revision
testCommentIdx = 0
rev = None
if (bug.whiteboard != None):
ret = re.compile('\[jsbugmon:([^\]]+)\]').search(bug.whiteboard)
if (ret != None and ret.groups > 1):
wbOpts = ret.group(1).split(",")
for opt in wbOpts:
if (opt.find("=") > 0):
(cmd,param) = opt.split('=')
if (cmd != None and param != None):
if (cmd == "origRev"):
rev = param
elif (cmd == "testComment" and param.isdigit()):
testCommentIdx = int(param)
# Look for the first comment
comment = bug.comments[testCommentIdx] if len(bug.comments) > testCommentIdx else None
if (comment == None):
raise BugException("Error: Specified bug does not have any comments")
text = comment.text
# Isolate revision to test for
if (rev == None):
rev = self.extractRevision(text)
else:
# Sanity check of the revision
rev = self.extractRevision(rev)
if (rev == None):
raise BugException("Error: Failed to isolate original revision for test")
opts = None
tipOpts = None
# Isolate options for testing, not explicitly instructed to guess
if not self.options.guessopts:
opts = self.extractOptions(text)
if (opts == None):
print "Warning: No options found, will try to guess"
arch = None
archList = None
if (bug.platform == "x86_64"):
arch = "64"
elif (bug.platform == "x86"):
arch = "32"
elif (bug.platform == "All"):
arch = "64"
archList = [ "64", "32" ] # TODO: Detect native platform here
else:
raise BugException("Error: Unsupported architecture \"" + bug.platform + "\" required by bug")
if (bug.version == "Trunk"):
reponame = "mozilla-central"
elif (bug.version == "Other Branch"):
reponame = "ionmonkey"
else:
raise BugException("Error: Unsupported branch \"" + bug.version + "\" required by bug")
if (tipBranch == None):
tipBranch = reponame
print "Repobase: " + self.repoBase
print "Reponame: " + reponame
repoDir = os.path.join(self.repoBase, reponame)
tipRepoDir = os.path.join(self.repoBase, tipBranch)
# We need at least some shell to extract the test from the bug,
# so we build a debug tip shell here already
updated = False
if not self.tipRev.has_key(repoDir):
# If we don't know the tip revision for this branch, update and get it
self.tipRev[repoDir] = self.hgUpdate(repoDir)
updated = True
(tipShell, tipRev) = self.getShell("cache/", arch, "dbg", 0, self.tipRev[repoDir], updated, repoDir)
# If the file already exists, then we can reuse it
if testCommentIdx > 0:
testFile = "bug" + str(bugnum) + "-" + str(testCommentIdx) + ".js"
else:
testFile = "bug" + str(bugnum) + ".js"
if (os.path.exists(testFile)):
print "Using existing (cached) testfile " + testFile
else:
# We need to detect where our test is.
blocks = text.split("\n\n")
found = False
cnt = 0
for i,block in enumerate(blocks):
# Write our test to file
outFile = open(testFile, "w")
outFile.write(block)
outFile.close()
(err, ret) = testBinary(tipShell, testFile, [], 0, timeout=30)
if (err.find("SyntaxError") < 0):
# We have found the test (or maybe only the start of the test)
# Try adding more code until we hit an error or are out of
# blocks.
oldBlock = block
curBlock = block
for j,block in enumerate(blocks):
if j > i:
curBlock = curBlock + "\n" + block
# Write our test to file
outFile = open(testFile, "w")
outFile.write(curBlock)
outFile.close()
(err, ret) = testBinary(tipShell, testFile, [], 0, timeout=30)
if (err.find("SyntaxError") >= 0):
# Too much, write oldBlock and break
outFile = open(testFile, "w")
outFile.write(oldBlock)
outFile.close()
break
else:
oldBlock = curBlock
found = True
print "Isolated possible testcase starting in textblock " + str(cnt)
break
cnt += 1
if not found:
# First try to find a suitable attachment
attachments = [a for a in bug.attachments if not bool(int(a.is_obsolete))]
for attachment in attachments:
# Seriously, we don't need anything larger than 512kb here^^
if (attachment.size <= 512*1024):
# Refetch attachment with content
url = urljoin(self.apiroot, 'attachment/%s/?%s&attachmentdata=1' % (attachment.id, self.bz.qs()))
attachment = attachment.get(url)
try:
rawData = base64.b64decode(attachment.data)
# Write our data to file
outFile = open(testFile, "w")
outFile.write(rawData)
outFile.close()
(err, ret) = testBinary(tipShell, testFile, [], 0, timeout=30)
if (err.find("SyntaxError") < 0):
# Found something that looks like JS :)
found = True
break
except TypeError:
pass
# If we still haven't found any test, give up here...
if not found:
# Ensure we don't cache the wrong test
os.remove(testFile)
raise BugException("Error: Failed to isolate test from comment")
(oouterr, oret) = (None, None)
(origShell, origRev) = (None, None)
# If we have an exact architecture, we will only test that
if (archList == None):
archList = [ arch ]
for compileType in ['dbg', 'opt']:
for archType in archList:
# Update to tip and cache result:
updated = False
if not self.tipRev.has_key(tipRepoDir):
# If we don't know the tip revision for this branch, update and get it
self.tipRev[tipRepoDir] = self.hgUpdate(tipRepoDir)
updated = True
(tipShell, tipRev) = self.getShell("cache/", archType, compileType, 0, self.tipRev[tipRepoDir], updated, tipRepoDir)
(origShell, origRev) = self.getShell("cache/", archType, compileType, 0, rev, False, repoDir)
if (opts != None):
(oouterr, oret) = testBinary(origShell, testFile, opts , 0, verbose=self.options.verbose, timeout=30)
else:
print "Guessing options...",
guessopts = self.guessopts[reponame]
for opt in guessopts:
topts = []
if opt == None:
print " no options",
else:
print " " + opt,
topts = opt.split(' ')
(oouterr, oret) = testBinary(origShell, testFile, topts , 0, verbose=self.options.verbose, timeout=30)
if (oret < 0):
opts = topts
break;
# If we reproduced with one arch, then we don't need to try the others
if (oret < 0):
break;
print ""
# If we reproduced with dbg, then we don't need to try opt
if (oret < 0):
break;
# Check if we reproduced at all (dbg or opt)
if (oret < 0):
print ""
print "Successfully reproduced bug (exit code " + str(oret) + ") on original revision " + rev + ":"
errl = oouterr.split("\n")
if len(errl) > 2: errl = errl[-2:]
for err in errl:
print err
if (opts != None):
# Try running on tip now
print "Testing bug on tip..."
if self.options.guessopts:
guessopts = self.guessopts[reponame]
for opt in guessopts:
tipOpts = []
if opt == None:
print " no options",
else:
print " " + opt,
tipOpts = opt.split(' ')
(touterr, tret) = testBinary(tipShell, testFile, tipOpts , 0, verbose=self.options.verbose, timeout=30)
if (tret < 0):
break;
else:
tipOpts = opts
(touterr, tret) = testBinary(tipShell, testFile, tipOpts , 0, verbose=self.options.verbose, timeout=30)
else:
print ""
if (tret < 0):
if (tret == oret):
if (opts == tipOpts):
print "Result: Bug still reproduces"
return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, compileType, BugMonitorResult.statusCodes.REPRODUCED_TIP)
else:
print "Result: Bug still reproduces, but with different options: " + " ".join(tipOpts) # TODO need another code here in the future
return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, compileType, BugMonitorResult.statusCodes.REPRODUCED_TIP)
else:
# Unlikely but possible, switched signal
print "Result: Bug now reproduces with signal " + str(tret) + " (previously " + str(oret) + ")"
return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, compileType, BugMonitorResult.statusCodes.REPRODUCED_SWITCHED)
else:
print "Result: Bug no longer reproduces"
return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, compileType, BugMonitorResult.statusCodes.REPRODUCED_FIXED)
else:
print "Error: Failed to reproduce bug on original revision"
return BugMonitorResult(reponame, rev, self.tipRev[tipRepoDir], opts, testFile, archType, compileType, BugMonitorResult.statusCodes.FAILED)
def postComment(self, bugnum, comment): url = urljoin(self.apiroot, 'bug/%s/comment?%s' % (bugnum, self.bz.qs())) return Comment(text=comment).post_to(url)
