def isDuplicate(r1: ReportWrapper, r2: ReportWrapper) -> DuplicateResult:
""" Returns a confidence rating on whether the two given reports are duplicates of each other """
for module in modules:
if (module.match(r1.getReportBody(), r1.getReportWeakness())
and # type: ignore
module.match(r2.getReportBody(),
r2.getReportWeakness())): # type: ignore
return sameCategoryIsDuplicate(
r1, r2, module.containsExploit) # type: ignore
return DuplicateResult((None, ID('A')))
def suggestPayout(report: ReportWrapper) -> Optional[BountyInfo]:
""" Returns a BountyInfo containing a suggested payout and the standard deviation for the given report """
if xss.match(report.getReportBody(), report.getReportWeakness()):
return suggestPayoutGivenType(config.payoutDB['xss'],
report.getVulnDomains())
if openRedirect.match(report.getReportBody(), report.getReportWeakness()):
return suggestPayoutGivenType(config.payoutDB['open redirect'],
report.getVulnDomains())
if sqli.match(report.getReportBody(), report.getReportWeakness()):
return suggestPayoutGivenType(config.payoutDB['sqli'],
report.getVulnDomains())
return None
def test_ReportWrapperGetters():
r = ReportWrapper(openRedirectReproJson)
assert r.getReportID() == '239981'
assert r.getLatestActivity() == ("blah open_redirect\n\n[some](http://example.com/redir.php?QUERY_STRING="
"https://google.com)")
assert r.getReportBody() == ("blah open_redirect\n\n[some](http://example.com/redir.php?QUERY_STRING="
"https://google.com)")
assert r.getReportWeakness() == "Open Redirect"
assert r.getReportTitle() == "open redirect"
assert r.getVulnDomains() == ['example.com']
r = ReportWrapper(openRedirectUnreproJson)
assert r.getReportID() == '240035'
assert r.getLatestActivity() == ("this is detected as an open redirect but there is no markdown link to it\n\n"
"https://example.com/redir.php?QUERY_STRING=https://google.com")
assert r.getReportBody() == ("this is detected as an open redirect but there is no markdown link to it\n\n"
"https://example.com/redir.php?QUERY_STRING=https://google.com")
assert r.getReportWeakness() == "Open Redirect"
assert r.getReportTitle() == "malformed open redirect"
assert r.getVulnDomains() == ['example.com']
def processReport(report: ReportWrapper, startTime: datetime) -> Optional[VulnTestInfo]:
""" Attempt to verify a given report """
if report.needsBotReply():
if startTime > report.getReportedTime():
return None
if config.DEBUG:
print("Processing %s" % report.getReportTitle())
for module in modules:
if module.match(report.getReportBody(), report.getReportWeakness()): # type: ignore
if config.DEBUG:
print(module.__file__.split('/')[-1] + " matched id=%s!" % report.getReportID())
vti = module.process(report) # type: ignore
if config.DEBUGVERBOSE:
print(vti)
if vti:
postComment(report.getReportID(), vti, addStopMessage=True)
if vti.reproduced and config.metadataLogging:
metadataVTI = generateMetadataVTI(report, vti)
postComment(report.getReportID(), metadataVTI, internal=True)
return vti
if config.DEBUG:
print("No matches")
return None
def sameCategoryIsDuplicate(r1: ReportWrapper, r2: ReportWrapper, containsExploit: Callable[[str], bool]) -> \
DuplicateResult:
""" Returns a confidence rating on whether the two given reports are duplicates of each other given that they are
of the same type of vulnerability and that containsExploit returns whether or not a given URL is exploiting
that class of vulnerability. """
# The links are the only things we refer to in our current duplicate detection algorithm
links1, links2 = getLinks(r1.getReportBody()), getLinks(r2.getReportBody())
malLinks1 = [
link for link in links1
if containsExploit(link) or containsExploit(unquote(link))
]
malLinks2 = [
link for link in links2
if containsExploit(link) or containsExploit(unquote(link))
]
if set(malLinks1) & set(malLinks2):
return DuplicateResult((99, ID('B')))
if set(links1) & set(links2):
return DuplicateResult((90, ID('C')))
parsedMalLinks1 = list(
filter(lambda n: n, map(AutoTriageUtils.parseURL, malLinks1)))
parsedMalLinks2 = list(
filter(lambda n: n, map(AutoTriageUtils.parseURL, malLinks2)))
parsedLinks1 = list(
filter(lambda n: n, map(AutoTriageUtils.parseURL, links1)))
parsedLinks2 = list(
filter(lambda n: n, map(AutoTriageUtils.parseURL, links2)))
malDomainParameterTuples1 = flatten([[(x.domain, x.path, key)
for key, val in x.queries.items()
if containsExploit(val)]
for x in parsedMalLinks1])
malDomainParameterTuples2 = flatten([[(x.domain, x.path, key)
for key, val in x.queries.items()
if containsExploit(val)]
for x in parsedMalLinks2])
parametersInCommon = (
set(flatten([parsed.queries.keys() for parsed in parsedLinks1]))
& set(flatten([parsed.queries.keys() for parsed in parsedLinks2])))
malParametersInCommon = (
set(flatten([parsed.queries.keys() for parsed in parsedMalLinks1]))
& set(flatten([parsed.queries.keys() for parsed in parsedMalLinks2])))
injectionParametersInCommon = (
set([param for domain, path, param in malDomainParameterTuples1])
& set([param for domain, path, param in malDomainParameterTuples2]))
malPathsInCommon = (set([
path for domain, path, param in malDomainParameterTuples1 if path != ''
]) & set([
path for domain, path, param in malDomainParameterTuples2 if path != ''
]))
pathsInCommon = (
set([parsed.path for parsed in parsedLinks1 if parsed.path != ''])
& set([parsed.path for parsed in parsedLinks2 if parsed.path != '']))
domains1 = set(
[x.domain for x in parsedLinks1 if '[server]' not in x.domain])
domains2 = set(
[x.domain for x in parsedLinks2 if '[server]' not in x.domain])
domainsInCommon = domains1 & domains2
malDomainsInCommon = (
set([x.domain
for x in parsedMalLinks1 if '[server]' not in x.domain]) &
set([x.domain for x in parsedMalLinks2 if '[server]' not in x.domain]))
return decide(len(malLinks1), len(malLinks2), len(parametersInCommon),
len(malParametersInCommon), len(pathsInCommon),
len(malPathsInCommon), len(domainsInCommon),
len(malDomainsInCommon), len(injectionParametersInCommon),
len(domains1 ^ domains2))
def processReport(report: ReportWrapper) -> bool:
""" Process a report via searching for duplicates and posting comments based off of the confidence levels
Returns whether or not the report was classified as a duplicate with a high confidence """
if report.getState() == "new" and not report.hasDuplicateComment(
) and not report.isVerified():
earlierReports = getAllOpenReports(
report.getReportedTime()) # type: List[ReportWrapper]
idConfTuples = [] # type: List[Tuple[str, int]]
matches = [] # type: List[str]
for earlierReport in earlierReports:
for module in modules:
if (module.match(report.getReportBody(),
report.getReportWeakness())
and # type: ignore
module.match(earlierReport.getReportBody(),
earlierReport.getReportWeakness())
): # type: ignore
matches.append(earlierReport.getReportID())
try:
confidence = int(isDuplicate(earlierReport, report)[0])
except TypeError:
confidence = 0
if confidence == 99:
AutoTriageUtils.postComment(
report.getReportID(),
VulnTestInfo(
message='Found a duplicate with 99%% confidence: #%s' %
earlierReport.getReportID(),
info={},
reproduced=False,
type=''),
internal=True)
if config.DEBUG:
print("Detected that %s (%s) is a duplicate of %s (%s)!" %
(report.getReportID(), report.getReportTitle(),
earlierReport.getReportID(),
earlierReport.getReportTitle()))
return False # Change to return True to make the bot stop interacting after finding a duplicate
elif confidence > 50:
idConfTuples.append((earlierReport.getReportID(), confidence))
# If you update the phrases here, you must also update them in AutoTriageUtils.ReportWrapper.hasDuplicateComment
if len(idConfTuples) > 0:
def idConfToStr(tuple: Tuple) -> str:
return (
'Detected a possible duplicate report with confidence of %s: #%s'
% (tuple[1], tuple[0]))
AutoTriageUtils.postComment(report.getReportID(),
VulnTestInfo(message='\n'.join([
idConfToStr(t)
for t in idConfTuples
]),
info={},
reproduced=False,
type=''),
internal=True)
if config.DEBUG:
print('Found partial matches: %s' % str(idConfTuples))
if len(matches) > 0 and len(matches) <= 5:
AutoTriageUtils.postComment(
report.getReportID(),
VulnTestInfo(message=(
'There are currently %s open reports about this type of '
'vulnerability: %s' %
(str(len(matches)), ', '.join(['#' + id
for id in matches]))),
info={},
reproduced=False,
type=''),
internal=True)
if config.DEBUG:
print(
'Found %s reports on the same type of vulnerability as %s: %s'
% (str(len(matches)), str(report.getReportID()), ', '.join(
['#' + id for id in matches])))
return False
def shouldProcess_match(report: ReportWrapper) -> bool:
""" Whether the bot should process the given ReportWrapper according to whether any of the modules match it """
return any([
m.match(report.getReportBody(), report.getReportWeakness())
for m in modules
]) # type: ignore
def test_match():
r = ReportWrapper(openRedirectReproJson)
assert openRedirect.match(r.getReportBody(), r.getReportWeakness())
r = ReportWrapper(openRedirectUnreproJson)
assert openRedirect.match(r.getReportBody(), r.getReportWeakness())