Example #1
0
def isDuplicate(r1: ReportWrapper, r2: ReportWrapper) -> DuplicateResult:
    """ Returns a confidence rating on whether the two given reports are duplicates of each other """
    for module in modules:
        if (module.match(r1.getReportBody(), r1.getReportWeakness())
                and  # type: ignore
                module.match(r2.getReportBody(),
                             r2.getReportWeakness())):  # type: ignore
            return sameCategoryIsDuplicate(
                r1, r2, module.containsExploit)  # type: ignore
    return DuplicateResult((None, ID('A')))
Example #2
0
def suggestPayout(report: ReportWrapper) -> Optional[BountyInfo]:
    """ Returns a BountyInfo containing a suggested payout and the standard deviation for the given report """
    if xss.match(report.getReportBody(), report.getReportWeakness()):
        return suggestPayoutGivenType(config.payoutDB['xss'],
                                      report.getVulnDomains())
    if openRedirect.match(report.getReportBody(), report.getReportWeakness()):
        return suggestPayoutGivenType(config.payoutDB['open redirect'],
                                      report.getVulnDomains())
    if sqli.match(report.getReportBody(), report.getReportWeakness()):
        return suggestPayoutGivenType(config.payoutDB['sqli'],
                                      report.getVulnDomains())
    return None
Example #3
0
def test_ReportWrapperGetters():
    r = ReportWrapper(openRedirectReproJson)
    assert r.getReportID() == '239981'
    assert r.getLatestActivity() == ("blah open_redirect\n\n[some](http://example.com/redir.php?QUERY_STRING="
                                     "https://google.com)")
    assert r.getReportBody() == ("blah open_redirect\n\n[some](http://example.com/redir.php?QUERY_STRING="
                                 "https://google.com)")
    assert r.getReportWeakness() == "Open Redirect"
    assert r.getReportTitle() == "open redirect"
    assert r.getVulnDomains() == ['example.com']
    r = ReportWrapper(openRedirectUnreproJson)
    assert r.getReportID() == '240035'
    assert r.getLatestActivity() == ("this is detected as an open redirect but there is no markdown link to it\n\n"
                                     "https://example.com/redir.php?QUERY_STRING=https://google.com")
    assert r.getReportBody() == ("this is detected as an open redirect but there is no markdown link to it\n\n"
                                 "https://example.com/redir.php?QUERY_STRING=https://google.com")
    assert r.getReportWeakness() == "Open Redirect"
    assert r.getReportTitle() == "malformed open redirect"
    assert r.getVulnDomains() == ['example.com']
Example #4
0
def processReport(report: ReportWrapper, startTime: datetime) -> Optional[VulnTestInfo]:
    """ Attempt to verify a given report """
    if report.needsBotReply():
        if startTime > report.getReportedTime():
            return None
        if config.DEBUG:
            print("Processing %s" % report.getReportTitle())
        for module in modules:
            if module.match(report.getReportBody(), report.getReportWeakness()):  # type: ignore
                if config.DEBUG:
                    print(module.__file__.split('/')[-1] + " matched id=%s!" % report.getReportID())
                vti = module.process(report)  # type: ignore
                if config.DEBUGVERBOSE:
                    print(vti)
                if vti:
                    postComment(report.getReportID(), vti, addStopMessage=True)
                    if vti.reproduced and config.metadataLogging:
                        metadataVTI = generateMetadataVTI(report, vti)
                        postComment(report.getReportID(), metadataVTI, internal=True)
                return vti
        if config.DEBUG:
            print("No matches")
    return None
Example #5
0
def sameCategoryIsDuplicate(r1: ReportWrapper, r2: ReportWrapper, containsExploit: Callable[[str], bool]) -> \
        DuplicateResult:
    """ Returns a confidence rating on whether the two given reports are duplicates of each other given that they are
        of the same type of vulnerability and that containsExploit returns whether or not a given URL is exploiting
        that class of vulnerability. """
    # The links are the only things we refer to in our current duplicate detection algorithm
    links1, links2 = getLinks(r1.getReportBody()), getLinks(r2.getReportBody())
    malLinks1 = [
        link for link in links1
        if containsExploit(link) or containsExploit(unquote(link))
    ]
    malLinks2 = [
        link for link in links2
        if containsExploit(link) or containsExploit(unquote(link))
    ]

    if set(malLinks1) & set(malLinks2):
        return DuplicateResult((99, ID('B')))
    if set(links1) & set(links2):
        return DuplicateResult((90, ID('C')))

    parsedMalLinks1 = list(
        filter(lambda n: n, map(AutoTriageUtils.parseURL, malLinks1)))
    parsedMalLinks2 = list(
        filter(lambda n: n, map(AutoTriageUtils.parseURL, malLinks2)))
    parsedLinks1 = list(
        filter(lambda n: n, map(AutoTriageUtils.parseURL, links1)))
    parsedLinks2 = list(
        filter(lambda n: n, map(AutoTriageUtils.parseURL, links2)))

    malDomainParameterTuples1 = flatten([[(x.domain, x.path, key)
                                          for key, val in x.queries.items()
                                          if containsExploit(val)]
                                         for x in parsedMalLinks1])
    malDomainParameterTuples2 = flatten([[(x.domain, x.path, key)
                                          for key, val in x.queries.items()
                                          if containsExploit(val)]
                                         for x in parsedMalLinks2])

    parametersInCommon = (
        set(flatten([parsed.queries.keys() for parsed in parsedLinks1]))
        & set(flatten([parsed.queries.keys() for parsed in parsedLinks2])))
    malParametersInCommon = (
        set(flatten([parsed.queries.keys() for parsed in parsedMalLinks1]))
        & set(flatten([parsed.queries.keys() for parsed in parsedMalLinks2])))

    injectionParametersInCommon = (
        set([param for domain, path, param in malDomainParameterTuples1])
        & set([param for domain, path, param in malDomainParameterTuples2]))

    malPathsInCommon = (set([
        path for domain, path, param in malDomainParameterTuples1 if path != ''
    ]) & set([
        path for domain, path, param in malDomainParameterTuples2 if path != ''
    ]))
    pathsInCommon = (
        set([parsed.path for parsed in parsedLinks1 if parsed.path != ''])
        & set([parsed.path for parsed in parsedLinks2 if parsed.path != '']))

    domains1 = set(
        [x.domain for x in parsedLinks1 if '[server]' not in x.domain])
    domains2 = set(
        [x.domain for x in parsedLinks2 if '[server]' not in x.domain])

    domainsInCommon = domains1 & domains2
    malDomainsInCommon = (
        set([x.domain
             for x in parsedMalLinks1 if '[server]' not in x.domain]) &
        set([x.domain for x in parsedMalLinks2 if '[server]' not in x.domain]))

    return decide(len(malLinks1), len(malLinks2), len(parametersInCommon),
                  len(malParametersInCommon), len(pathsInCommon),
                  len(malPathsInCommon), len(domainsInCommon),
                  len(malDomainsInCommon), len(injectionParametersInCommon),
                  len(domains1 ^ domains2))
Example #6
0
def processReport(report: ReportWrapper) -> bool:
    """ Process a report via searching for duplicates and posting comments based off of the confidence levels
          Returns whether or not the report was classified as a duplicate with a high confidence """
    if report.getState() == "new" and not report.hasDuplicateComment(
    ) and not report.isVerified():
        earlierReports = getAllOpenReports(
            report.getReportedTime())  # type: List[ReportWrapper]
        idConfTuples = []  # type: List[Tuple[str, int]]
        matches = []  # type: List[str]
        for earlierReport in earlierReports:
            for module in modules:
                if (module.match(report.getReportBody(),
                                 report.getReportWeakness())
                        and  # type: ignore
                        module.match(earlierReport.getReportBody(),
                                     earlierReport.getReportWeakness())
                    ):  # type: ignore
                    matches.append(earlierReport.getReportID())
            try:
                confidence = int(isDuplicate(earlierReport, report)[0])
            except TypeError:
                confidence = 0
            if confidence == 99:
                AutoTriageUtils.postComment(
                    report.getReportID(),
                    VulnTestInfo(
                        message='Found a duplicate with 99%% confidence: #%s' %
                        earlierReport.getReportID(),
                        info={},
                        reproduced=False,
                        type=''),
                    internal=True)
                if config.DEBUG:
                    print("Detected that %s (%s) is a duplicate of %s (%s)!" %
                          (report.getReportID(), report.getReportTitle(),
                           earlierReport.getReportID(),
                           earlierReport.getReportTitle()))
                return False  # Change to return True to make the bot stop interacting after finding a duplicate
            elif confidence > 50:
                idConfTuples.append((earlierReport.getReportID(), confidence))
        # If you update the phrases here, you must also update them in AutoTriageUtils.ReportWrapper.hasDuplicateComment
        if len(idConfTuples) > 0:

            def idConfToStr(tuple: Tuple) -> str:
                return (
                    'Detected a possible duplicate report with confidence of %s: #%s'
                    % (tuple[1], tuple[0]))

            AutoTriageUtils.postComment(report.getReportID(),
                                        VulnTestInfo(message='\n'.join([
                                            idConfToStr(t)
                                            for t in idConfTuples
                                        ]),
                                                     info={},
                                                     reproduced=False,
                                                     type=''),
                                        internal=True)
            if config.DEBUG:
                print('Found partial matches: %s' % str(idConfTuples))
        if len(matches) > 0 and len(matches) <= 5:
            AutoTriageUtils.postComment(
                report.getReportID(),
                VulnTestInfo(message=(
                    'There are currently %s open reports about this type of '
                    'vulnerability: %s' %
                    (str(len(matches)), ', '.join(['#' + id
                                                   for id in matches]))),
                             info={},
                             reproduced=False,
                             type=''),
                internal=True)
            if config.DEBUG:
                print(
                    'Found %s reports on the same type of vulnerability as %s: %s'
                    % (str(len(matches)), str(report.getReportID()), ', '.join(
                        ['#' + id for id in matches])))
    return False
Example #7
0
def shouldProcess_match(report: ReportWrapper) -> bool:
    """ Whether the bot should process the given ReportWrapper according to whether any of the modules match it """
    return any([
        m.match(report.getReportBody(), report.getReportWeakness())
        for m in modules
    ])  # type: ignore
Example #8
0
def test_match():
    r = ReportWrapper(openRedirectReproJson)
    assert openRedirect.match(r.getReportBody(), r.getReportWeakness())
    r = ReportWrapper(openRedirectUnreproJson)
    assert openRedirect.match(r.getReportBody(), r.getReportWeakness())