def isDuplicate(r1: ReportWrapper, r2: ReportWrapper) -> DuplicateResult: """ Returns a confidence rating on whether the two given reports are duplicates of each other """ for module in modules: if (module.match(r1.getReportBody(), r1.getReportWeakness()) and # type: ignore module.match(r2.getReportBody(), r2.getReportWeakness())): # type: ignore return sameCategoryIsDuplicate( r1, r2, module.containsExploit) # type: ignore return DuplicateResult((None, ID('A')))
def suggestPayout(report: ReportWrapper) -> Optional[BountyInfo]: """ Returns a BountyInfo containing a suggested payout and the standard deviation for the given report """ if xss.match(report.getReportBody(), report.getReportWeakness()): return suggestPayoutGivenType(config.payoutDB['xss'], report.getVulnDomains()) if openRedirect.match(report.getReportBody(), report.getReportWeakness()): return suggestPayoutGivenType(config.payoutDB['open redirect'], report.getVulnDomains()) if sqli.match(report.getReportBody(), report.getReportWeakness()): return suggestPayoutGivenType(config.payoutDB['sqli'], report.getVulnDomains()) return None
def test_ReportWrapperGetters(): r = ReportWrapper(openRedirectReproJson) assert r.getReportID() == '239981' assert r.getLatestActivity() == ("blah open_redirect\n\n[some](http://example.com/redir.php?QUERY_STRING=" "https://google.com)") assert r.getReportBody() == ("blah open_redirect\n\n[some](http://example.com/redir.php?QUERY_STRING=" "https://google.com)") assert r.getReportWeakness() == "Open Redirect" assert r.getReportTitle() == "open redirect" assert r.getVulnDomains() == ['example.com'] r = ReportWrapper(openRedirectUnreproJson) assert r.getReportID() == '240035' assert r.getLatestActivity() == ("this is detected as an open redirect but there is no markdown link to it\n\n" "https://example.com/redir.php?QUERY_STRING=https://google.com") assert r.getReportBody() == ("this is detected as an open redirect but there is no markdown link to it\n\n" "https://example.com/redir.php?QUERY_STRING=https://google.com") assert r.getReportWeakness() == "Open Redirect" assert r.getReportTitle() == "malformed open redirect" assert r.getVulnDomains() == ['example.com']
def processReport(report: ReportWrapper, startTime: datetime) -> Optional[VulnTestInfo]: """ Attempt to verify a given report """ if report.needsBotReply(): if startTime > report.getReportedTime(): return None if config.DEBUG: print("Processing %s" % report.getReportTitle()) for module in modules: if module.match(report.getReportBody(), report.getReportWeakness()): # type: ignore if config.DEBUG: print(module.__file__.split('/')[-1] + " matched id=%s!" % report.getReportID()) vti = module.process(report) # type: ignore if config.DEBUGVERBOSE: print(vti) if vti: postComment(report.getReportID(), vti, addStopMessage=True) if vti.reproduced and config.metadataLogging: metadataVTI = generateMetadataVTI(report, vti) postComment(report.getReportID(), metadataVTI, internal=True) return vti if config.DEBUG: print("No matches") return None
def sameCategoryIsDuplicate(r1: ReportWrapper, r2: ReportWrapper, containsExploit: Callable[[str], bool]) -> \ DuplicateResult: """ Returns a confidence rating on whether the two given reports are duplicates of each other given that they are of the same type of vulnerability and that containsExploit returns whether or not a given URL is exploiting that class of vulnerability. """ # The links are the only things we refer to in our current duplicate detection algorithm links1, links2 = getLinks(r1.getReportBody()), getLinks(r2.getReportBody()) malLinks1 = [ link for link in links1 if containsExploit(link) or containsExploit(unquote(link)) ] malLinks2 = [ link for link in links2 if containsExploit(link) or containsExploit(unquote(link)) ] if set(malLinks1) & set(malLinks2): return DuplicateResult((99, ID('B'))) if set(links1) & set(links2): return DuplicateResult((90, ID('C'))) parsedMalLinks1 = list( filter(lambda n: n, map(AutoTriageUtils.parseURL, malLinks1))) parsedMalLinks2 = list( filter(lambda n: n, map(AutoTriageUtils.parseURL, malLinks2))) parsedLinks1 = list( filter(lambda n: n, map(AutoTriageUtils.parseURL, links1))) parsedLinks2 = list( filter(lambda n: n, map(AutoTriageUtils.parseURL, links2))) malDomainParameterTuples1 = flatten([[(x.domain, x.path, key) for key, val in x.queries.items() if containsExploit(val)] for x in parsedMalLinks1]) malDomainParameterTuples2 = flatten([[(x.domain, x.path, key) for key, val in x.queries.items() if containsExploit(val)] for x in parsedMalLinks2]) parametersInCommon = ( set(flatten([parsed.queries.keys() for parsed in parsedLinks1])) & set(flatten([parsed.queries.keys() for parsed in parsedLinks2]))) malParametersInCommon = ( set(flatten([parsed.queries.keys() for parsed in parsedMalLinks1])) & set(flatten([parsed.queries.keys() for parsed in parsedMalLinks2]))) injectionParametersInCommon = ( set([param for domain, path, param in malDomainParameterTuples1]) & set([param for domain, path, param in malDomainParameterTuples2])) malPathsInCommon = (set([ path for domain, path, param in malDomainParameterTuples1 if path != '' ]) & set([ path for domain, path, param in malDomainParameterTuples2 if path != '' ])) pathsInCommon = ( set([parsed.path for parsed in parsedLinks1 if parsed.path != '']) & set([parsed.path for parsed in parsedLinks2 if parsed.path != ''])) domains1 = set( [x.domain for x in parsedLinks1 if '[server]' not in x.domain]) domains2 = set( [x.domain for x in parsedLinks2 if '[server]' not in x.domain]) domainsInCommon = domains1 & domains2 malDomainsInCommon = ( set([x.domain for x in parsedMalLinks1 if '[server]' not in x.domain]) & set([x.domain for x in parsedMalLinks2 if '[server]' not in x.domain])) return decide(len(malLinks1), len(malLinks2), len(parametersInCommon), len(malParametersInCommon), len(pathsInCommon), len(malPathsInCommon), len(domainsInCommon), len(malDomainsInCommon), len(injectionParametersInCommon), len(domains1 ^ domains2))
def processReport(report: ReportWrapper) -> bool: """ Process a report via searching for duplicates and posting comments based off of the confidence levels Returns whether or not the report was classified as a duplicate with a high confidence """ if report.getState() == "new" and not report.hasDuplicateComment( ) and not report.isVerified(): earlierReports = getAllOpenReports( report.getReportedTime()) # type: List[ReportWrapper] idConfTuples = [] # type: List[Tuple[str, int]] matches = [] # type: List[str] for earlierReport in earlierReports: for module in modules: if (module.match(report.getReportBody(), report.getReportWeakness()) and # type: ignore module.match(earlierReport.getReportBody(), earlierReport.getReportWeakness()) ): # type: ignore matches.append(earlierReport.getReportID()) try: confidence = int(isDuplicate(earlierReport, report)[0]) except TypeError: confidence = 0 if confidence == 99: AutoTriageUtils.postComment( report.getReportID(), VulnTestInfo( message='Found a duplicate with 99%% confidence: #%s' % earlierReport.getReportID(), info={}, reproduced=False, type=''), internal=True) if config.DEBUG: print("Detected that %s (%s) is a duplicate of %s (%s)!" % (report.getReportID(), report.getReportTitle(), earlierReport.getReportID(), earlierReport.getReportTitle())) return False # Change to return True to make the bot stop interacting after finding a duplicate elif confidence > 50: idConfTuples.append((earlierReport.getReportID(), confidence)) # If you update the phrases here, you must also update them in AutoTriageUtils.ReportWrapper.hasDuplicateComment if len(idConfTuples) > 0: def idConfToStr(tuple: Tuple) -> str: return ( 'Detected a possible duplicate report with confidence of %s: #%s' % (tuple[1], tuple[0])) AutoTriageUtils.postComment(report.getReportID(), VulnTestInfo(message='\n'.join([ idConfToStr(t) for t in idConfTuples ]), info={}, reproduced=False, type=''), internal=True) if config.DEBUG: print('Found partial matches: %s' % str(idConfTuples)) if len(matches) > 0 and len(matches) <= 5: AutoTriageUtils.postComment( report.getReportID(), VulnTestInfo(message=( 'There are currently %s open reports about this type of ' 'vulnerability: %s' % (str(len(matches)), ', '.join(['#' + id for id in matches]))), info={}, reproduced=False, type=''), internal=True) if config.DEBUG: print( 'Found %s reports on the same type of vulnerability as %s: %s' % (str(len(matches)), str(report.getReportID()), ', '.join( ['#' + id for id in matches]))) return False
def shouldProcess_match(report: ReportWrapper) -> bool: """ Whether the bot should process the given ReportWrapper according to whether any of the modules match it """ return any([ m.match(report.getReportBody(), report.getReportWeakness()) for m in modules ]) # type: ignore
def test_match(): r = ReportWrapper(openRedirectReproJson) assert openRedirect.match(r.getReportBody(), r.getReportWeakness()) r = ReportWrapper(openRedirectUnreproJson) assert openRedirect.match(r.getReportBody(), r.getReportWeakness())