Пример #1
0
    def setupReports(self):
        if self.arguments.outputFile is not None:
            outputFile = FileUtils.get_abs_path(self.arguments.outputFile)
            self.output.outputFile(outputFile)
        else:
            if self.batch:
                fileName = "BATCH"
                fileName += self.getOutputExtension()
                directoryPath = self.batchDirectoryPath
            else:
                localRequester = Requester(self.urlList[0])
                fileName = ('{}_'.format(
                    localRequester.basePath.replace(os.path.sep, ".")[:-1]))
                fileName += time.strftime('%y-%m-%d_%H-%M-%S')
                fileName += self.getOutputExtension()
                directoryPath = FileUtils.build_path(self.savePath, 'reports',
                                                     localRequester.host)

            outputFile = FileUtils.build_path(directoryPath, fileName)

            if FileUtils.exists(outputFile):
                i = 2

                while FileUtils.exists(outputFile + "_" + str(i)):
                    i += 1

                outputFile += "_" + str(i)

            if not FileUtils.exists(directoryPath):
                FileUtils.create_directory(directoryPath)

                if not FileUtils.exists(directoryPath):
                    self.output.error(
                        "Couldn't create the reports folder at {}".format(
                            directoryPath))
                    sys.exit(1)

            self.output.outputFile(outputFile)

        if self.arguments.outputFile and self.arguments.outputFormat:
            self.reportManager = ReportManager(self.arguments.outputFormat,
                                               self.arguments.outputFile)
        elif self.arguments.outputFormat:
            self.reportManager = ReportManager(self.arguments.outputFormat,
                                               outputFile)
        else:
            self.reportManager = ReportManager("plain", outputFile)
Пример #2
0
    def setup_reports(self):
        if self.arguments.output_file:
            output_file = FileUtils.get_abs_path(self.arguments.output_file)
            self.output.output_file(output_file)
        else:
            if self.batch:
                file_name = "BATCH"
                file_name += self.get_output_extension()
                directory_path = self.batch_directory_path
            else:
                local_requester = Requester(self.url_list[0])
                file_name = ("{}_".format(
                    local_requester.base_path.replace(os.path.sep, ".")[:-1]))
                file_name += time.strftime("%y-%m-%d_%H-%M-%S")
                file_name += self.get_output_extension()
                directory_path = FileUtils.build_path(self.save_path,
                                                      local_requester.host)

            output_file = FileUtils.build_path(directory_path, file_name)

            if FileUtils.exists(output_file):
                i = 2

                while FileUtils.exists(output_file + "_" + str(i)):
                    i += 1

                output_file += "_" + str(i)

            if not FileUtils.exists(directory_path):
                FileUtils.create_directory(directory_path)

                if not FileUtils.exists(directory_path):
                    self.output.error(
                        "Couldn't create the reports folder at {}".format(
                            directory_path))
                    sys.exit(1)

            self.output.output_file(output_file)

        if self.arguments.output_file and self.arguments.output_format:
            self.report_manager = ReportManager(self.arguments.output_format,
                                                self.arguments.output_file)
        elif self.arguments.output_format:
            self.report_manager = ReportManager(self.arguments.output_format,
                                                output_file)
        else:
            self.report_manager = ReportManager("plain", output_file)
Пример #3
0
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = (open(
            FileUtils.buildPath(script_path, "lib", "controller",
                                "banner.txt")).read().format(**VERSION))

        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.savePath = self.script_path
        self.doneDirs = []

        self.recursive_level_max = self.arguments.recursive_level_max

        if self.arguments.httpmethod.lower() not in [
                "get", "head", "post", "put", "patch", "options", "delete",
                "trace", "debug"
        ]:
            self.output.error("Invalid HTTP method!")
            exit(1)

        self.httpmethod = self.arguments.httpmethod.lower()

        if self.arguments.saveHome:
            savePath = self.getSavePath()

            if not FileUtils.exists(savePath):
                FileUtils.createDirectory(savePath)

            if FileUtils.exists(savePath) and not FileUtils.isDir(savePath):
                self.output.error(
                    "Cannot use {} because is a file. Should be a directory".
                    format(savePath))
                exit(1)

            if not FileUtils.canWrite(savePath):
                self.output.error(
                    "Directory {} is not writable".format(savePath))
                exit(1)

            logs = FileUtils.buildPath(savePath, "logs")

            if not FileUtils.exists(logs):
                FileUtils.createDirectory(logs)

            reports = FileUtils.buildPath(savePath, "reports")

            if not FileUtils.exists(reports):
                FileUtils.createDirectory(reports)

            self.savePath = savePath

        self.reportsPath = FileUtils.buildPath(self.savePath, "logs")
        self.blacklists = self.getBlacklists()
        self.includeStatusCodes = self.arguments.includeStatusCodes
        self.excludeStatusCodes = self.arguments.excludeStatusCodes
        self.excludeTexts = self.arguments.excludeTexts
        self.excludeRegexps = self.arguments.excludeRegexps
        self.recursive = self.arguments.recursive
        self.suppressEmpty = self.arguments.suppressEmpty
        self.minimumResponseSize = self.arguments.minimumResponseSize
        self.maximumResponseSize = self.arguments.maximumResponseSize
        self.directories = Queue()
        self.excludeSubdirs = (arguments.excludeSubdirs
                               if arguments.excludeSubdirs else [])

        self.dictionary = Dictionary(
            self.arguments.wordlist, self.arguments.extensions,
            self.arguments.suffixes, self.arguments.prefixes,
            self.arguments.lowercase, self.arguments.uppercase,
            self.arguments.capitalization, self.arguments.forceExtensions,
            self.arguments.noDotExtensions, self.arguments.excludeExtensions,
            self.arguments.noExtension)

        self.errorLog = None
        self.errorLogPath = None
        self.threadsLock = Lock()
        self.batch = False
        self.batchSession = None
        self.currentJob = 0
        self.allJobs = 0

        self.output.header(program_banner)
        self.printConfig()
        self.setupErrorLogs()
        self.output.errorLogFile(self.errorLogPath)

        if self.arguments.autoSave and len(self.arguments.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(
                self.batchDirectoryPath))

        if self.arguments.useRandomAgents:
            self.randomAgents = FileUtils.getLines(
                FileUtils.buildPath(script_path, "db", "user-agents.txt"))

        try:
            for url in list(dict.fromkeys(self.arguments.urlList)):
                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url
                    self.output.setTarget(self.currentUrl)

                    try:
                        self.requester = Requester(
                            url,
                            cookie=self.arguments.cookie,
                            useragent=self.arguments.useragent,
                            maxPool=self.arguments.threadsCount,
                            maxRetries=self.arguments.maxRetries,
                            delay=self.arguments.delay,
                            timeout=self.arguments.timeout,
                            ip=self.arguments.ip,
                            proxy=self.arguments.proxy,
                            proxylist=self.arguments.proxylist,
                            redirect=self.arguments.redirect,
                            requestByHostname=self.arguments.requestByHostname,
                            httpmethod=self.httpmethod,
                            data=self.arguments.data,
                        )

                        self.requester.request("")

                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                    if self.arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    for key, value in arguments.headers.items():
                        self.requester.setHeader(key, value)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath

                    if self.arguments.scanSubdirs:
                        for subdir in self.arguments.scanSubdirs:
                            self.directories.put(subdir)
                            self.allJobs += 1

                    else:
                        self.directories.put("")
                        self.allJobs += 1

                    self.setupReports(self.requester)

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        testFailPath=self.arguments.testFailPath,
                        threads=self.arguments.threadsCount,
                        matchCallbacks=matchCallbacks,
                        notFoundCallbacks=notFoundCallbacks,
                        errorCallbacks=errorCallbacks,
                    )
                    try:
                        self.wait()
                    except RequestException as e:
                        self.output.error(
                            "Fatal error during site scanning: " +
                            e.args[0]["message"])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    continue

        except KeyboardInterrupt:
            self.output.error("\nCanceled by the user")
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

            self.reportManager.close()

        self.output.warning("\nTask Completed")
Пример #4
0
class Controller(object):
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = (open(
            FileUtils.buildPath(script_path, "lib", "controller",
                                "banner.txt")).read().format(**VERSION))

        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.savePath = self.script_path
        self.doneDirs = []

        self.recursive_level_max = self.arguments.recursive_level_max

        if self.arguments.httpmethod.lower() not in [
                "get", "head", "post", "put", "patch", "options", "delete",
                "trace", "debug"
        ]:
            self.output.error("Invalid HTTP method!")
            exit(1)

        self.httpmethod = self.arguments.httpmethod.lower()

        if self.arguments.saveHome:
            savePath = self.getSavePath()

            if not FileUtils.exists(savePath):
                FileUtils.createDirectory(savePath)

            if FileUtils.exists(savePath) and not FileUtils.isDir(savePath):
                self.output.error(
                    "Cannot use {} because is a file. Should be a directory".
                    format(savePath))
                exit(1)

            if not FileUtils.canWrite(savePath):
                self.output.error(
                    "Directory {} is not writable".format(savePath))
                exit(1)

            logs = FileUtils.buildPath(savePath, "logs")

            if not FileUtils.exists(logs):
                FileUtils.createDirectory(logs)

            reports = FileUtils.buildPath(savePath, "reports")

            if not FileUtils.exists(reports):
                FileUtils.createDirectory(reports)

            self.savePath = savePath

        self.reportsPath = FileUtils.buildPath(self.savePath, "logs")
        self.blacklists = self.getBlacklists()
        self.includeStatusCodes = self.arguments.includeStatusCodes
        self.excludeStatusCodes = self.arguments.excludeStatusCodes
        self.excludeTexts = self.arguments.excludeTexts
        self.excludeRegexps = self.arguments.excludeRegexps
        self.recursive = self.arguments.recursive
        self.suppressEmpty = self.arguments.suppressEmpty
        self.minimumResponseSize = self.arguments.minimumResponseSize
        self.maximumResponseSize = self.arguments.maximumResponseSize
        self.directories = Queue()
        self.excludeSubdirs = (arguments.excludeSubdirs
                               if arguments.excludeSubdirs else [])

        self.dictionary = Dictionary(
            self.arguments.wordlist, self.arguments.extensions,
            self.arguments.suffixes, self.arguments.prefixes,
            self.arguments.lowercase, self.arguments.uppercase,
            self.arguments.capitalization, self.arguments.forceExtensions,
            self.arguments.noDotExtensions, self.arguments.excludeExtensions,
            self.arguments.noExtension)

        self.errorLog = None
        self.errorLogPath = None
        self.threadsLock = Lock()
        self.batch = False
        self.batchSession = None
        self.currentJob = 0
        self.allJobs = 0

        self.output.header(program_banner)
        self.printConfig()
        self.setupErrorLogs()
        self.output.errorLogFile(self.errorLogPath)

        if self.arguments.autoSave and len(self.arguments.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(
                self.batchDirectoryPath))

        if self.arguments.useRandomAgents:
            self.randomAgents = FileUtils.getLines(
                FileUtils.buildPath(script_path, "db", "user-agents.txt"))

        try:
            for url in list(dict.fromkeys(self.arguments.urlList)):
                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url
                    self.output.setTarget(self.currentUrl)

                    try:
                        self.requester = Requester(
                            url,
                            cookie=self.arguments.cookie,
                            useragent=self.arguments.useragent,
                            maxPool=self.arguments.threadsCount,
                            maxRetries=self.arguments.maxRetries,
                            delay=self.arguments.delay,
                            timeout=self.arguments.timeout,
                            ip=self.arguments.ip,
                            proxy=self.arguments.proxy,
                            proxylist=self.arguments.proxylist,
                            redirect=self.arguments.redirect,
                            requestByHostname=self.arguments.requestByHostname,
                            httpmethod=self.httpmethod,
                            data=self.arguments.data,
                        )

                        self.requester.request("")

                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                    if self.arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    for key, value in arguments.headers.items():
                        self.requester.setHeader(key, value)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath

                    if self.arguments.scanSubdirs:
                        for subdir in self.arguments.scanSubdirs:
                            self.directories.put(subdir)
                            self.allJobs += 1

                    else:
                        self.directories.put("")
                        self.allJobs += 1

                    self.setupReports(self.requester)

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        testFailPath=self.arguments.testFailPath,
                        threads=self.arguments.threadsCount,
                        matchCallbacks=matchCallbacks,
                        notFoundCallbacks=notFoundCallbacks,
                        errorCallbacks=errorCallbacks,
                    )
                    try:
                        self.wait()
                    except RequestException as e:
                        self.output.error(
                            "Fatal error during site scanning: " +
                            e.args[0]["message"])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    continue

        except KeyboardInterrupt:
            self.output.error("\nCanceled by the user")
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

            self.reportManager.close()

        self.output.warning("\nTask Completed")

    def printConfig(self):

        self.output.config(
            ', '.join(self.arguments.extensions),
            ', '.join(self.arguments.prefixes),
            ', '.join(self.arguments.suffixes),
            str(self.arguments.threadsCount),
            str(len(self.dictionary)),
            str(self.httpmethod),
            self.recursive,
            str(self.recursive_level_max),
        )

    def getSavePath(self):
        basePath = None
        dirPath = None
        basePath = os.path.expanduser("~")

        if os.name == "nt":
            dirPath = "dirsearch"
        else:
            dirPath = ".dirsearch"

        return FileUtils.buildPath(basePath, dirPath)

    def getBlacklists(self):
        reext = re.compile('\%ext\%', re.IGNORECASE)
        reextdot = re.compile('\.\%ext\%', re.IGNORECASE)
        blacklists = {}

        for status in [400, 403, 500]:
            blacklistFileName = FileUtils.buildPath(self.script_path, "db")
            blacklistFileName = FileUtils.buildPath(
                blacklistFileName, "{}_blacklist.txt".format(status))

            if not FileUtils.canRead(blacklistFileName):
                # Skip if cannot read file
                continue

            blacklists[status] = []

            for line in FileUtils.getLines(blacklistFileName):
                # Skip comments
                if line.lstrip().startswith("#"):
                    continue

                # The same with Dictionary.py
                if line.startswith("/"):
                    line = line[1:]

                # Classic dirsearch blacklist processing (with %EXT% keyword)
                if "%ext%" in line.lower():
                    for extension in self.arguments.extensions:
                        if self.arguments.noDotExtensions:
                            entry = reextdot.sub(extension, line)

                        else:
                            entry = line

                        entry = reext.sub(extension, entry)

                        blacklists[status].append(entry)

                # Forced extensions is not used here because -r is only used for wordlist (in documentation),
                # applying in blacklist may create false negatives

                else:
                    blacklists[status].append(line)

        return blacklists

    def setupErrorLogs(self):
        fileName = "errors-{0}.log".format(time.strftime("%y-%m-%d_%H-%M-%S"))
        self.errorLogPath = FileUtils.buildPath(
            FileUtils.buildPath(self.savePath, "logs", fileName))
        self.errorLog = open(self.errorLogPath, "w")

    def setupBatchReports(self):
        self.batch = True
        self.batchSession = "BATCH-{0}".format(
            time.strftime("%y-%m-%d_%H-%M-%S"))
        self.batchDirectoryPath = FileUtils.buildPath(self.savePath, "reports",
                                                      self.batchSession)

        if not FileUtils.exists(self.batchDirectoryPath):
            FileUtils.createDirectory(self.batchDirectoryPath)

            if not FileUtils.exists(self.batchDirectoryPath):
                self.output.error("Couldn't create batch folder {}".format(
                    self.batchDirectoryPath))
                sys.exit(1)

        if FileUtils.canWrite(self.batchDirectoryPath):
            FileUtils.createDirectory(self.batchDirectoryPath)
            targetsFile = FileUtils.buildPath(self.batchDirectoryPath,
                                              "TARGETS.txt")
            FileUtils.writeLines(targetsFile, self.arguments.urlList)

        else:
            self.output.error("Couldn't create batch folder {}.".format(
                self.batchDirectoryPath))
            sys.exit(1)

    def setupReports(self, requester):
        if self.arguments.autoSave:

            basePath = "/" if not (len(
                requester.basePath)) else requester.basePath
            basePath = basePath.replace(os.path.sep, ".")[:-1]
            fileName = None
            directoryPath = None

            if self.batch:
                fileName = requester.host
                directoryPath = self.batchDirectoryPath

            else:

                fileName = ('{}_'.format(basePath))
                fileName += time.strftime('%y-%m-%d_%H-%M-%S.txt')
                directoryPath = FileUtils.buildPath(self.savePath, 'reports',
                                                    requester.host)

            outputFile = FileUtils.buildPath(directoryPath, fileName)

            self.output.outputFile(outputFile)

            if FileUtils.exists(outputFile):
                i = 2

                while FileUtils.exists(outputFile + "_" + str(i)):
                    i += 1

                outputFile += "_" + str(i)

            if not FileUtils.exists(directoryPath):
                FileUtils.createDirectory(directoryPath)

                if not FileUtils.exists(directoryPath):
                    self.output.error(
                        "Couldn't create reports folder {}".format(
                            directoryPath))
                    sys.exit(1)
            if FileUtils.canWrite(directoryPath):
                report = None

                if self.arguments.autoSaveFormat == "simple":
                    report = SimpleReport(requester.host, requester.port,
                                          requester.protocol,
                                          requester.basePath, outputFile,
                                          self.batch)
                if self.arguments.autoSaveFormat == "json":
                    report = JSONReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                    )
                else:
                    report = PlainTextReport(requester.host, requester.port,
                                             requester.protocol,
                                             requester.basePath, outputFile,
                                             self.batch)

                self.reportManager.addOutput(report)

            else:
                self.output.error(
                    "Can't write reports to {}".format(directoryPath))
                sys.exit(1)

        # TODO: format, refactor code
        if self.arguments.simpleOutputFile:
            self.reportManager.addOutput(
                SimpleReport(requester.host, requester.port,
                             requester.protocol, requester.basePath,
                             self.arguments.simpleOutputFile, self.batch))

        if self.arguments.plainTextOutputFile:
            self.reportManager.addOutput(
                PlainTextReport(requester.host, requester.port,
                                requester.protocol, requester.basePath,
                                self.arguments.plainTextOutputFile,
                                self.batch))

        if self.arguments.jsonOutputFile:
            self.reportManager.addOutput(
                JSONReport(requester.host, requester.port, requester.protocol,
                           requester.basePath, self.arguments.jsonOutputFile,
                           self.batch))

    # TODO: Refactor, this function should be a decorator for all the filters
    def matchCallback(self, path):
        self.index += 1

        if path.status:

            if path.status not in self.excludeStatusCodes and (
                    not self.includeStatusCodes
                    or path.status in self.includeStatusCodes
            ) and (not (self.blacklists.get(path.status)) or path.path
                   not in self.blacklists.get(path.status)) and not (
                       self.suppressEmpty and (not (len(path.response.body)))
                   ) and not (
                       (self.minimumResponseSize and
                        self.minimumResponseSize > len(path.response.body)) or
                       (self.maximumResponseSize and
                        self.maximumResponseSize < len(path.response.body))):

                for excludeText in self.excludeTexts:
                    if excludeText in path.response.body.decode():
                        del path
                        return

                for excludeRegexp in self.excludeRegexps:

                    if (re.search(excludeRegexp, path.response.body.decode())
                            is not None):
                        del path
                        return

                pathIsInScanSubdirs = False
                addedToQueue = False

                if self.arguments.scanSubdirs:
                    for subdir in self.arguments.scanSubdirs:
                        if subdir == path.path + "/":
                            pathIsInScanSubdirs = True

                if not pathIsInScanSubdirs:
                    if not self.recursive:
                        pass
                    elif path.response.redirect:
                        addedToQueue = self.addRedirectDirectory(path)
                    else:
                        addedToQueue = self.addDirectory(path.path)

                self.output.statusReport(path.path, path.response,
                                         self.arguments.full_url, addedToQueue)

                newPath = "{}{}".format(self.currentDirectory, path.path)

                self.reportManager.addPath(newPath, path.status, path.response)

                self.reportManager.save()

                del path

    def notFoundCallback(self, path):
        self.index += 1
        self.output.lastPath(path, self.index, len(self.dictionary),
                             self.currentJob, self.allJobs)
        del path

    def errorCallback(self, path, errorMsg):
        self.output.addConnectionError()
        del path

    def appendErrorLog(self, path, errorMsg):
        with self.threadsLock:
            line = time.strftime("[%y-%m-%d %H:%M:%S] - ")
            line += self.currentUrl + " - " + path + " - " + errorMsg
            self.errorLog.write(os.linesep + line)
            self.errorLog.flush()

    def handleInterrupt(self):
        self.output.warning("CTRL+C detected: Pausing threads, please wait...")
        self.fuzzer.pause()

        try:
            while True:
                msg = "[e]xit / [c]ontinue"

                if not self.directories.empty():
                    msg += " / [n]ext"

                if len(self.arguments.urlList) > 1:
                    msg += " / [s]kip target"

                self.output.inLine(msg + ": ")

                option = input()

                if option.lower() == "e":
                    self.exit = True
                    self.fuzzer.stop()
                    raise KeyboardInterrupt

                elif option.lower() == "c":
                    self.fuzzer.play()
                    return

                elif not self.directories.empty() and option.lower() == "n":
                    self.fuzzer.stop()
                    return

                elif len(self.arguments.urlList) > 1 and option.lower() == "s":
                    raise SkipTargetInterrupt

                else:
                    continue

        except KeyboardInterrupt as SystemExit:
            self.exit = True
            raise KeyboardInterrupt

    def processPaths(self):
        while True:
            try:
                while not self.fuzzer.wait(0.3):
                    continue
                break

            except (KeyboardInterrupt, SystemExit) as e:
                self.handleInterrupt()

    def wait(self):
        while not self.directories.empty():
            self.currentJob += 1
            gc.collect()
            self.index = 0
            self.currentDirectory = self.directories.get()
            self.output.warning("[{1}] Starting: {0}".format(
                self.currentDirectory, time.strftime("%H:%M:%S")))
            self.fuzzer.requester.basePath = self.basePath + self.currentDirectory
            self.output.basePath = self.basePath + self.currentDirectory
            self.fuzzer.start()
            self.processPaths()

        return

    def addDirectory(self, path):
        if path.endswith("/"):
            if path in [directory + "/" for directory in self.excludeSubdirs]:
                return False

            dir = self.currentDirectory + path

            if dir in self.doneDirs:
                return False

            if dir.count("/") > self.recursive_level_max:
                return False

            self.directories.put(dir)
            self.allJobs += 1

            self.doneDirs.append(dir)

            return True

        else:
            return False

    def addRedirectDirectory(self, path):
        # Resolve the redirect header relative to the current URL and add the
        # path to self.directories if it is a subdirectory of the current URL

        baseUrl = self.currentUrl.rstrip("/") + "/" + self.currentDirectory

        baseUrl = baseUrl.rstrip("/") + "/"

        absoluteUrl = urllib.parse.urljoin(baseUrl, path.response.redirect)

        if (absoluteUrl.startswith(baseUrl) and absoluteUrl != baseUrl
                and absoluteUrl.endswith("/")):
            dir = absoluteUrl[len(self.currentUrl.rstrip("/")) + 1:]

            if dir in self.doneDirs:
                return False

            if dir.count("/") > self.recursive_level_max:
                return False

            self.directories.put(dir)
            self.allJobs += 1

            self.doneDirs.append(dir)

            return True

        return False
Пример #5
0
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = open(
            FileUtils.buildPath(script_path, "lib", "controller",
                                "banner.txt")).read().format(**VERSION)

        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.savePath = self.script_path

        if self.arguments.saveHome:
            savePath = self.getSavePath()

            if not FileUtils.exists(savePath):
                FileUtils.createDirectory(savePath)

            if FileUtils.exists(savePath) and not FileUtils.isDir(savePath):
                self.output.error(
                    'Cannot use {} because is a file. Should be a directory'.
                    format(savePath))
                exit(1)

            if not FileUtils.canWrite(savePath):
                self.output.error(
                    'Directory {} is not writable'.format(savePath))
                exit(1)

            logs = FileUtils.buildPath(savePath, "logs")

            if not FileUtils.exists(logs):
                FileUtils.createDirectory(logs)

            reports = FileUtils.buildPath(savePath, "reports")

            if not FileUtils.exists(reports):
                FileUtils.createDirectory(reports)

            self.savePath = savePath

        self.reportsPath = FileUtils.buildPath(self.savePath, "logs")
        self.blacklists = self.getBlacklists()
        self.fuzzer = None
        self.excludeStatusCodes = self.arguments.excludeStatusCodes
        self.recursive = self.arguments.recursive
        self.suppressEmpty = self.arguments.suppressEmpty
        self.directories = Queue()
        self.excludeSubdirs = (arguments.excludeSubdirs
                               if arguments.excludeSubdirs is not None else [])
        self.output.header(program_banner)
        self.dictionary = Dictionary(self.arguments.wordlist,
                                     self.arguments.extensions,
                                     self.arguments.lowercase,
                                     self.arguments.forceExtensions)
        self.printConfig()
        self.errorLog = None
        self.errorLogPath = None
        self.errorLogLock = Lock()
        self.batch = False
        self.batchSession = None
        self.setupErrorLogs()
        self.output.newLine("\nError Log: {0}".format(self.errorLogPath))

        if self.arguments.autoSave and len(self.arguments.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(
                self.batchDirectoryPath))

        if self.arguments.useRandomAgents:
            self.randomAgents = FileUtils.getLines(
                FileUtils.buildPath(script_path, "db", "user-agents.txt"))

        try:
            for url in self.arguments.urlList:

                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url
                    self.output.target(self.currentUrl)

                    try:
                        self.requester = Requester(
                            url,
                            cookie=self.arguments.cookie,
                            useragent=self.arguments.useragent,
                            maxPool=self.arguments.threadsCount,
                            maxRetries=self.arguments.maxRetries,
                            delay=self.arguments.delay,
                            timeout=self.arguments.timeout,
                            ip=self.arguments.ip,
                            proxy=self.arguments.proxy,
                            redirect=self.arguments.redirect,
                            requestByHostname=self.arguments.requestByHostname)
                        self.requester.request("/")

                    except RequestException as e:
                        self.output.error(e.args[0]['message'])
                        raise SkipTargetInterrupt

                    if self.arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    for key, value in arguments.headers.items():
                        self.requester.setHeader(key, value)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath

                    if self.arguments.scanSubdirs is not None:
                        for subdir in self.arguments.scanSubdirs:
                            self.directories.put(subdir)

                    else:
                        self.directories.put('')

                    self.setupReports(self.requester)

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        testFailPath=self.arguments.testFailPath,
                        threads=self.arguments.threadsCount,
                        matchCallbacks=matchCallbacks,
                        notFoundCallbacks=notFoundCallbacks,
                        errorCallbacks=errorCallbacks)
                    try:
                        self.wait()
                    except RequestException as e:
                        self.output.error(
                            "Fatal error during site scanning: " +
                            e.args[0]['message'])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    continue

                finally:
                    self.reportManager.save()

        except KeyboardInterrupt:
            self.output.error('\nCanceled by the user')
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

            self.reportManager.close()

        self.output.warning('\nTask Completed')
Пример #6
0
class Controller(object):
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = open(
            FileUtils.buildPath(script_path, "lib", "controller",
                                "banner.txt")).read().format(**VERSION)

        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.savePath = self.script_path

        if self.arguments.saveHome:
            savePath = self.getSavePath()

            if not FileUtils.exists(savePath):
                FileUtils.createDirectory(savePath)

            if FileUtils.exists(savePath) and not FileUtils.isDir(savePath):
                self.output.error(
                    'Cannot use {} because is a file. Should be a directory'.
                    format(savePath))
                exit(1)

            if not FileUtils.canWrite(savePath):
                self.output.error(
                    'Directory {} is not writable'.format(savePath))
                exit(1)

            logs = FileUtils.buildPath(savePath, "logs")

            if not FileUtils.exists(logs):
                FileUtils.createDirectory(logs)

            reports = FileUtils.buildPath(savePath, "reports")

            if not FileUtils.exists(reports):
                FileUtils.createDirectory(reports)

            self.savePath = savePath

        self.reportsPath = FileUtils.buildPath(self.savePath, "logs")
        self.blacklists = self.getBlacklists()
        self.fuzzer = None
        self.excludeStatusCodes = self.arguments.excludeStatusCodes
        self.recursive = self.arguments.recursive
        self.suppressEmpty = self.arguments.suppressEmpty
        self.directories = Queue()
        self.excludeSubdirs = (arguments.excludeSubdirs
                               if arguments.excludeSubdirs is not None else [])
        self.output.header(program_banner)
        self.dictionary = Dictionary(self.arguments.wordlist,
                                     self.arguments.extensions,
                                     self.arguments.lowercase,
                                     self.arguments.forceExtensions)
        self.printConfig()
        self.errorLog = None
        self.errorLogPath = None
        self.errorLogLock = Lock()
        self.batch = False
        self.batchSession = None
        self.setupErrorLogs()
        self.output.newLine("\nError Log: {0}".format(self.errorLogPath))

        if self.arguments.autoSave and len(self.arguments.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(
                self.batchDirectoryPath))

        if self.arguments.useRandomAgents:
            self.randomAgents = FileUtils.getLines(
                FileUtils.buildPath(script_path, "db", "user-agents.txt"))

        try:
            for url in self.arguments.urlList:

                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url
                    self.output.target(self.currentUrl)

                    try:
                        self.requester = Requester(
                            url,
                            cookie=self.arguments.cookie,
                            useragent=self.arguments.useragent,
                            maxPool=self.arguments.threadsCount,
                            maxRetries=self.arguments.maxRetries,
                            delay=self.arguments.delay,
                            timeout=self.arguments.timeout,
                            ip=self.arguments.ip,
                            proxy=self.arguments.proxy,
                            redirect=self.arguments.redirect,
                            requestByHostname=self.arguments.requestByHostname)
                        self.requester.request("/")

                    except RequestException as e:
                        self.output.error(e.args[0]['message'])
                        raise SkipTargetInterrupt

                    if self.arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    for key, value in arguments.headers.items():
                        self.requester.setHeader(key, value)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath

                    if self.arguments.scanSubdirs is not None:
                        for subdir in self.arguments.scanSubdirs:
                            self.directories.put(subdir)

                    else:
                        self.directories.put('')

                    self.setupReports(self.requester)

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        testFailPath=self.arguments.testFailPath,
                        threads=self.arguments.threadsCount,
                        matchCallbacks=matchCallbacks,
                        notFoundCallbacks=notFoundCallbacks,
                        errorCallbacks=errorCallbacks)
                    try:
                        self.wait()
                    except RequestException as e:
                        self.output.error(
                            "Fatal error during site scanning: " +
                            e.args[0]['message'])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    continue

                finally:
                    self.reportManager.save()

        except KeyboardInterrupt:
            self.output.error('\nCanceled by the user')
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

            self.reportManager.close()

        self.output.warning('\nTask Completed')

    def printConfig(self):
        self.output.config(', '.join(self.arguments.extensions),
                           str(self.arguments.threadsCount),
                           str(len(self.dictionary)))

    def getSavePath(self):
        basePath = None
        dirPath = None
        basePath = os.path.expanduser('~')

        if os.name == 'nt':
            dirPath = "dirsearch"
        else:
            dirPath = ".dirsearch"

        return FileUtils.buildPath(basePath, dirPath)

    def getBlacklists(self):
        blacklists = {}

        for status in [400, 403, 500]:
            blacklistFileName = FileUtils.buildPath(self.script_path, 'db')
            blacklistFileName = FileUtils.buildPath(
                blacklistFileName, '{}_blacklist.txt'.format(status))

            if not FileUtils.canRead(blacklistFileName):
                # Skip if cannot read file
                continue

            blacklists[status] = []

            for line in FileUtils.getLines(blacklistFileName):
                # Skip comments
                if line.lstrip().startswith('#'):
                    continue

                blacklists[status].append(line)

        return blacklists

    def setupErrorLogs(self):
        fileName = "errors-{0}.log".format(time.strftime('%y-%m-%d_%H-%M-%S'))
        self.errorLogPath = FileUtils.buildPath(
            FileUtils.buildPath(self.savePath, "logs", fileName))
        self.errorLog = open(self.errorLogPath, "w")

    def setupBatchReports(self):
        self.batch = True
        self.batchSession = "BATCH-{0}".format(
            time.strftime('%y-%m-%d_%H-%M-%S'))
        self.batchDirectoryPath = FileUtils.buildPath(self.savePath, "reports",
                                                      self.batchSession)

        if not FileUtils.exists(self.batchDirectoryPath):
            FileUtils.createDirectory(self.batchDirectoryPath)

            if not FileUtils.exists(self.batchDirectoryPath):
                self.output.error("Couldn't create batch folder {}".format(
                    self.batchDirectoryPath))
                sys.exit(1)

        if FileUtils.canWrite(self.batchDirectoryPath):
            FileUtils.createDirectory(self.batchDirectoryPath)
            targetsFile = FileUtils.buildPath(self.batchDirectoryPath,
                                              "TARGETS.txt")
            FileUtils.writeLines(targetsFile, self.arguments.urlList)

        else:
            self.output.error("Couldn't create batch folder {}.".format(
                self.batchDirectoryPath))
            sys.exit(1)

    def setupReports(self, requester):
        if self.arguments.autoSave:
            basePath = ('/'
                        if requester.basePath is '' else requester.basePath)
            basePath = basePath.replace(os.path.sep, '.')[1:-1]
            fileName = None
            directoryPath = None

            if self.batch:
                fileName = requester.host
                directoryPath = self.batchDirectoryPath

            else:
                fileName = ('{}_'.format(basePath)
                            if basePath is not '' else '')
                fileName += time.strftime('%y-%m-%d_%H-%M-%S')
                directoryPath = FileUtils.buildPath(self.savePath, 'reports',
                                                    requester.host)

            outputFile = FileUtils.buildPath(directoryPath, fileName)

            if FileUtils.exists(outputFile):
                i = 2

                while FileUtils.exists(outputFile + "_" + str(i)):
                    i += 1

                outputFile += "_" + str(i)

            if not FileUtils.exists(directoryPath):
                FileUtils.createDirectory(directoryPath)

                if not FileUtils.exists(directoryPath):
                    self.output.error(
                        "Couldn't create reports folder {}".format(
                            directoryPath))
                    sys.exit(1)
            if FileUtils.canWrite(directoryPath):
                report = None

                if self.arguments.autoSaveFormat == 'simple':
                    report = SimpleReport(requester.host, requester.port,
                                          requester.protocol,
                                          requester.basePath, outputFile)
                if self.arguments.autoSaveFormat == 'json':
                    report = JSONReport(requester.host, requester.port,
                                        requester.protocol, requester.basePath,
                                        outputFile)
                else:
                    report = PlainTextReport(requester.host, requester.port,
                                             requester.protocol,
                                             requester.basePath, outputFile)

                self.reportManager.addOutput(report)

            else:
                self.output.error(
                    "Can't write reports to {}".format(directoryPath))
                sys.exit(1)

        if self.arguments.simpleOutputFile is not None:
            self.reportManager.addOutput(
                SimpleReport(requester.host, requester.port,
                             requester.protocol, requester.basePath,
                             self.arguments.simpleOutputFile))

        if self.arguments.plainTextOutputFile is not None:
            self.reportManager.addOutput(
                PlainTextReport(requester.host, requester.port,
                                requester.protocol, requester.basePath,
                                self.arguments.plainTextOutputFile))

        if self.arguments.jsonOutputFile is not None:
            self.reportManager.addOutput(
                JSONReport(requester.host, requester.port, requester.protocol,
                           requester.basePath, self.arguments.jsonOutputFile))

    def matchCallback(self, path):
        self.index += 1

        if path.status is not None:
            if path.status not in self.excludeStatusCodes and (
                    self.blacklists.get(path.status) is None
                    or path.path not in self.blacklists.get(
                        path.status)) and not (self.suppressEmpty and
                                               (len(path.response.body) == 0)):
                self.output.statusReport(path.path, path.response)
                self.addDirectory(path.path)
                self.reportManager.addPath(self.currentDirectory + path.path,
                                           path.status, path.response)
                self.reportManager.save()
                del path

    def notFoundCallback(self, path):
        self.index += 1
        self.output.lastPath(path, self.index, len(self.dictionary))
        del path

    def errorCallback(self, path, errorMsg):
        self.output.addConnectionError()
        del path

    def appendErrorLog(self, path, errorMsg):
        with self.errorLogLock:
            line = time.strftime('[%y-%m-%d %H:%M:%S] - ')
            line += self.currentUrl + " - " + path + " - " + errorMsg
            self.errorLog.write(os.linesep + line)
            self.errorLog.flush()

    def handleInterrupt(self):
        self.output.warning('CTRL+C detected: Pausing threads, please wait...')
        self.fuzzer.pause()

        try:
            while True:
                msg = "[e]xit / [c]ontinue"

                if not self.directories.empty():
                    msg += " / [n]ext"

                if len(self.arguments.urlList) > 1:
                    msg += " / [s]kip target"

                self.output.inLine(msg + ': ')

                option = input()

                if option.lower() == 'e':
                    self.exit = True
                    self.fuzzer.stop()
                    raise KeyboardInterrupt

                elif option.lower() == 'c':
                    self.fuzzer.play()
                    return

                elif not self.directories.empty() and option.lower() == 'n':
                    self.fuzzer.stop()
                    return

                elif len(self.arguments.urlList) > 1 and option.lower() == 's':
                    raise SkipTargetInterrupt

                else:
                    continue

        except KeyboardInterrupt as SystemExit:
            self.exit = True
            raise KeyboardInterrupt

    def processPaths(self):
        while True:
            try:
                while not self.fuzzer.wait(0.3):
                    continue
                break

            except (KeyboardInterrupt, SystemExit) as e:
                self.handleInterrupt()

    def wait(self):
        while not self.directories.empty():
            self.index = 0
            self.currentDirectory = self.directories.get()
            self.output.warning('[{1}] Starting: {0}'.format(
                self.currentDirectory, time.strftime('%H:%M:%S')))
            self.fuzzer.requester.basePath = self.basePath + self.currentDirectory
            self.output.basePath = self.basePath + self.currentDirectory
            self.fuzzer.start()
            self.processPaths()
        return

    def addDirectory(self, path):
        if not self.recursive:
            return False

        if path.endswith('/'):
            if path in [directory + '/' for directory in self.excludeSubdirs]:
                return False

            self.directories.put(self.currentDirectory + path)
            return True

        else:
            return False
Пример #7
0
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = (open(
            FileUtils.build_path(script_path, "lib", "controller",
                                 "banner.txt")).read().format(**VERSION))

        self.directories = Queue()
        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.savePath = self.script_path
        self.doneDirs = []

        if arguments.raw_file:
            # Overwrite python-requests default headers
            default_headers = {
                "User-Agent": None,
                "Accept-Encoding": None,
                "Accept": None,
            }

            _raw = Raw(arguments.raw_file, arguments.scheme)
            self.urlList = [_raw.url()]
            self.httpmethod = _raw.method()
            self.data = _raw.data()
            self.headers = {**default_headers, **_raw.headers()}
            self.cookie = _raw.cookie()
            self.useragent = _raw.user_agent()
        else:
            default_headers = {
                "User-Agent":
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
                "Accept-Language": "*",
                "Accept-Encoding": "*",
                "Keep-Alive": "300",
                "Cache-Control": "max-age=0",
            }

            self.urlList = list(filter(None, dict.fromkeys(arguments.urlList)))
            self.httpmethod = arguments.httpmethod.lower()
            self.data = arguments.data
            self.headers = {**default_headers, **arguments.headers}
            self.cookie = arguments.cookie
            self.useragent = arguments.useragent

        self.recursion_depth = arguments.recursion_depth

        if arguments.saveHome:
            savePath = self.getSavePath()

            if not FileUtils.exists(savePath):
                FileUtils.create_directory(savePath)

            if FileUtils.exists(savePath) and not FileUtils.is_dir(savePath):
                self.output.error(
                    "Cannot use {} because it's a file. Should be a directory".
                    format(savePath))
                exit(1)

            if not FileUtils.can_write(savePath):
                self.output.error(
                    "Directory {} is not writable".format(savePath))
                exit(1)

            logs = FileUtils.build_path(savePath, "logs")

            if not FileUtils.exists(logs):
                FileUtils.create_directory(logs)

            reports = FileUtils.build_path(savePath, "reports")

            if not FileUtils.exists(reports):
                FileUtils.create_directory(reports)

            self.savePath = savePath

        self.reportsPath = FileUtils.build_path(self.savePath, "logs")
        self.blacklists = self.getBlacklists()
        self.includeStatusCodes = arguments.includeStatusCodes
        self.excludeStatusCodes = arguments.excludeStatusCodes
        self.excludeSizes = arguments.excludeSizes
        self.excludeTexts = arguments.excludeTexts
        self.excludeRegexps = arguments.excludeRegexps
        self.excludeRedirects = arguments.excludeRedirects
        self.recursive = arguments.recursive
        self.minimumResponseSize = arguments.minimumResponseSize
        self.maximumResponseSize = arguments.maximumResponseSize
        self.scanSubdirs = arguments.scanSubdirs
        self.excludeSubdirs = (arguments.excludeSubdirs
                               if arguments.excludeSubdirs else [])

        self.dictionary = Dictionary(
            arguments.wordlist, arguments.extensions, arguments.suffixes,
            arguments.prefixes, arguments.lowercase, arguments.uppercase,
            arguments.capitalization, arguments.forceExtensions,
            arguments.excludeExtensions, arguments.noExtension,
            arguments.onlySelected)

        self.allJobs = len(self.scanSubdirs) if self.scanSubdirs else 1
        self.currentJob = 0
        self.errorLog = None
        self.errorLogPath = None
        self.threadsLock = Lock()
        self.batch = False
        self.batchSession = None
        self.skip429 = False

        self.output.header(program_banner)
        self.printConfig()
        self.setupErrorLogs()
        self.output.errorLogFile(self.errorLogPath)

        if arguments.autoSave and len(self.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(
                self.batchDirectoryPath))

        if arguments.useRandomAgents:
            self.randomAgents = FileUtils.get_lines(
                FileUtils.build_path(script_path, "db", "user-agents.txt"))

        try:
            for url in self.urlList:
                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url if url.endswith("/") else url + "/"
                    self.output.setTarget(self.currentUrl,
                                          self.arguments.scheme)

                    try:
                        self.requester = Requester(
                            url,
                            cookie=self.cookie,
                            useragent=self.useragent,
                            maxPool=arguments.threadsCount,
                            maxRetries=arguments.maxRetries,
                            timeout=arguments.timeout,
                            ip=arguments.ip,
                            proxy=arguments.proxy,
                            proxylist=arguments.proxylist,
                            redirect=arguments.redirect,
                            requestByHostname=arguments.requestByHostname,
                            httpmethod=self.httpmethod,
                            data=self.data,
                            scheme=arguments.scheme,
                        )

                        for key, value in self.headers.items():
                            self.requester.setHeader(key, value)

                        self.requester.request("")

                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                    if arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath

                    if self.scanSubdirs:
                        for subdir in self.scanSubdirs:
                            self.directories.put(subdir)

                    else:
                        self.directories.put("")

                    self.setupReports(self.requester)

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        testFailPath=arguments.testFailPath,
                        threads=arguments.threadsCount,
                        delay=arguments.delay,
                        matchCallbacks=matchCallbacks,
                        notFoundCallbacks=notFoundCallbacks,
                        errorCallbacks=errorCallbacks,
                    )
                    try:
                        self.prepare()
                    except RequestException as e:
                        self.output.error("Fatal error during scanning: " +
                                          e.args[0]["message"])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    continue

        except KeyboardInterrupt:
            self.output.error("\nCanceled by the user")
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

            self.reportManager.close()

        self.output.warning("\nTask Completed")
Пример #8
0
class Controller(object):
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = (open(
            FileUtils.build_path(script_path, "lib", "controller",
                                 "banner.txt")).read().format(**VERSION))

        self.directories = Queue()
        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.savePath = self.script_path
        self.doneDirs = []

        if arguments.raw_file:
            # Overwrite python-requests default headers
            default_headers = {
                "User-Agent": None,
                "Accept-Encoding": None,
                "Accept": None,
            }

            _raw = Raw(arguments.raw_file, arguments.scheme)
            self.urlList = [_raw.url()]
            self.httpmethod = _raw.method()
            self.data = _raw.data()
            self.headers = {**default_headers, **_raw.headers()}
            self.cookie = _raw.cookie()
            self.useragent = _raw.user_agent()
        else:
            default_headers = {
                "User-Agent":
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
                "Accept-Language": "*",
                "Accept-Encoding": "*",
                "Keep-Alive": "300",
                "Cache-Control": "max-age=0",
            }

            self.urlList = list(filter(None, dict.fromkeys(arguments.urlList)))
            self.httpmethod = arguments.httpmethod.lower()
            self.data = arguments.data
            self.headers = {**default_headers, **arguments.headers}
            self.cookie = arguments.cookie
            self.useragent = arguments.useragent

        self.recursion_depth = arguments.recursion_depth

        if arguments.saveHome:
            savePath = self.getSavePath()

            if not FileUtils.exists(savePath):
                FileUtils.create_directory(savePath)

            if FileUtils.exists(savePath) and not FileUtils.is_dir(savePath):
                self.output.error(
                    "Cannot use {} because it's a file. Should be a directory".
                    format(savePath))
                exit(1)

            if not FileUtils.can_write(savePath):
                self.output.error(
                    "Directory {} is not writable".format(savePath))
                exit(1)

            logs = FileUtils.build_path(savePath, "logs")

            if not FileUtils.exists(logs):
                FileUtils.create_directory(logs)

            reports = FileUtils.build_path(savePath, "reports")

            if not FileUtils.exists(reports):
                FileUtils.create_directory(reports)

            self.savePath = savePath

        self.reportsPath = FileUtils.build_path(self.savePath, "logs")
        self.blacklists = self.getBlacklists()
        self.includeStatusCodes = arguments.includeStatusCodes
        self.excludeStatusCodes = arguments.excludeStatusCodes
        self.excludeSizes = arguments.excludeSizes
        self.excludeTexts = arguments.excludeTexts
        self.excludeRegexps = arguments.excludeRegexps
        self.excludeRedirects = arguments.excludeRedirects
        self.recursive = arguments.recursive
        self.minimumResponseSize = arguments.minimumResponseSize
        self.maximumResponseSize = arguments.maximumResponseSize
        self.scanSubdirs = arguments.scanSubdirs
        self.excludeSubdirs = (arguments.excludeSubdirs
                               if arguments.excludeSubdirs else [])

        self.dictionary = Dictionary(
            arguments.wordlist, arguments.extensions, arguments.suffixes,
            arguments.prefixes, arguments.lowercase, arguments.uppercase,
            arguments.capitalization, arguments.forceExtensions,
            arguments.excludeExtensions, arguments.noExtension,
            arguments.onlySelected)

        self.allJobs = len(self.scanSubdirs) if self.scanSubdirs else 1
        self.currentJob = 0
        self.errorLog = None
        self.errorLogPath = None
        self.threadsLock = Lock()
        self.batch = False
        self.batchSession = None
        self.skip429 = False

        self.output.header(program_banner)
        self.printConfig()
        self.setupErrorLogs()
        self.output.errorLogFile(self.errorLogPath)

        if arguments.autoSave and len(self.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(
                self.batchDirectoryPath))

        if arguments.useRandomAgents:
            self.randomAgents = FileUtils.get_lines(
                FileUtils.build_path(script_path, "db", "user-agents.txt"))

        try:
            for url in self.urlList:
                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url if url.endswith("/") else url + "/"
                    self.output.setTarget(self.currentUrl,
                                          self.arguments.scheme)

                    try:
                        self.requester = Requester(
                            url,
                            cookie=self.cookie,
                            useragent=self.useragent,
                            maxPool=arguments.threadsCount,
                            maxRetries=arguments.maxRetries,
                            timeout=arguments.timeout,
                            ip=arguments.ip,
                            proxy=arguments.proxy,
                            proxylist=arguments.proxylist,
                            redirect=arguments.redirect,
                            requestByHostname=arguments.requestByHostname,
                            httpmethod=self.httpmethod,
                            data=self.data,
                            scheme=arguments.scheme,
                        )

                        for key, value in self.headers.items():
                            self.requester.setHeader(key, value)

                        self.requester.request("")

                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                    if arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath

                    if self.scanSubdirs:
                        for subdir in self.scanSubdirs:
                            self.directories.put(subdir)

                    else:
                        self.directories.put("")

                    self.setupReports(self.requester)

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        testFailPath=arguments.testFailPath,
                        threads=arguments.threadsCount,
                        delay=arguments.delay,
                        matchCallbacks=matchCallbacks,
                        notFoundCallbacks=notFoundCallbacks,
                        errorCallbacks=errorCallbacks,
                    )
                    try:
                        self.prepare()
                    except RequestException as e:
                        self.output.error("Fatal error during scanning: " +
                                          e.args[0]["message"])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    continue

        except KeyboardInterrupt:
            self.output.error("\nCanceled by the user")
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

            self.reportManager.close()

        self.output.warning("\nTask Completed")

    def printConfig(self):
        self.output.config(
            ', '.join(self.arguments.extensions),
            ', '.join(self.arguments.prefixes),
            ', '.join(self.arguments.suffixes),
            str(self.arguments.threadsCount),
            str(len(self.dictionary)),
            str(self.httpmethod),
        )

    def getSavePath(self):
        basePath = None
        dirPath = None
        basePath = os.path.expanduser("~")

        if os.name == "nt":
            dirPath = "dirsearch"
        else:
            dirPath = ".dirsearch"

        return FileUtils.build_path(basePath, dirPath)

    def getBlacklists(self):
        reext = re.compile(r'\%ext\%', re.IGNORECASE)
        blacklists = {}

        for status in [400, 403, 500]:
            blacklistFileName = FileUtils.build_path(self.script_path, "db")
            blacklistFileName = FileUtils.build_path(
                blacklistFileName, "{}_blacklist.txt".format(status))

            if not FileUtils.can_read(blacklistFileName):
                # Skip if cannot read file
                continue

            blacklists[status] = []

            for line in FileUtils.get_lines(blacklistFileName):
                # Skip comments
                if line.lstrip().startswith("#"):
                    continue

                if line.startswith("/"):
                    line = line[1:]

                # Classic dirsearch blacklist processing (with %EXT% keyword)
                if "%ext%" in line.lower():
                    for extension in self.arguments.extensions:
                        entry = reext.sub(extension, line)

                        blacklists[status].append(entry)

                # Forced extensions is not used here because -r is only used for wordlist,
                # applying in blacklist may create false negatives

                else:
                    blacklists[status].append(line)

        return blacklists

    def setupErrorLogs(self):
        fileName = "errors-{0}.log".format(time.strftime("%y-%m-%d_%H-%M-%S"))
        self.errorLogPath = FileUtils.build_path(
            FileUtils.build_path(self.savePath, "logs", fileName))

        try:
            self.errorLog = open(self.errorLogPath, "w")
        except PermissionError:
            self.output.error(
                "Couldn't create the error log. Try running again with highest permission"
            )
            sys.exit(1)

    def setupBatchReports(self):
        self.batch = True
        self.batchSession = "BATCH-{0}".format(
            time.strftime("%y-%m-%d_%H-%M-%S"))
        self.batchDirectoryPath = FileUtils.build_path(self.savePath,
                                                       "reports",
                                                       self.batchSession)

        if not FileUtils.exists(self.batchDirectoryPath):
            FileUtils.create_directory(self.batchDirectoryPath)

            if not FileUtils.exists(self.batchDirectoryPath):
                self.output.error("Couldn't create batch folder at {}".format(
                    self.batchDirectoryPath))
                sys.exit(1)

        if FileUtils.can_write(self.batchDirectoryPath):
            FileUtils.create_directory(self.batchDirectoryPath)
            targetsFile = FileUtils.build_path(self.batchDirectoryPath,
                                               "TARGETS.txt")
            FileUtils.write_lines(targetsFile, self.urlList)

        else:
            self.output.error("Couldn't create batch folder at {}".format(
                self.batchDirectoryPath))
            sys.exit(1)

    def setupReports(self, requester):
        if self.arguments.autoSave:

            basePath = requester.basePath
            basePath = basePath.replace(os.path.sep, ".")[:-1]
            fileName = None
            directoryPath = None

            if self.batch:
                fileName = requester.host
                directoryPath = self.batchDirectoryPath

            else:

                fileName = ('{}_'.format(basePath))
                fileName += time.strftime('%y-%m-%d_%H-%M-%S')
                fileName += ".{0}".format(self.arguments.autoSaveFormat)
                directoryPath = FileUtils.build_path(self.savePath, 'reports',
                                                     requester.host)

            outputFile = FileUtils.build_path(directoryPath, fileName)

            self.output.outputFile(outputFile)

            if FileUtils.exists(outputFile):
                i = 2

                while FileUtils.exists(outputFile + "_" + str(i)):
                    i += 1

                outputFile += "_" + str(i)

            if not FileUtils.exists(directoryPath):
                FileUtils.create_directory(directoryPath)

                if not FileUtils.exists(directoryPath):
                    self.output.error(
                        "Couldn't create the reports folder at {}".format(
                            directoryPath))
                    sys.exit(1)
            if FileUtils.can_write(directoryPath):
                report = None

                if self.arguments.autoSaveFormat == "simple":
                    report = SimpleReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch,
                    )
                elif self.arguments.autoSaveFormat == "json":
                    report = JSONReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch,
                    )
                elif self.arguments.autoSaveFormat == "xml":
                    report = XMLReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch,
                    )
                elif self.arguments.autoSaveFormat == "md":
                    report = MarkdownReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch,
                    )
                elif self.arguments.autoSaveFormat == "csv":
                    report = CSVReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch,
                    )
                else:
                    report = PlainTextReport(requester.host, requester.port,
                                             requester.protocol,
                                             requester.basePath, outputFile,
                                             self.batch)

                self.reportManager.addOutput(report)

            else:
                self.output.error(
                    "Can't write reports to {}".format(directoryPath))
                sys.exit(1)

        # TODO: format, refactor code
        if self.arguments.simpleOutputFile:
            self.reportManager.addOutput(
                SimpleReport(requester.host, requester.port,
                             requester.protocol, requester.basePath,
                             self.arguments.simpleOutputFile, self.batch))

        if self.arguments.plainTextOutputFile:
            self.reportManager.addOutput(
                PlainTextReport(requester.host, requester.port,
                                requester.protocol, requester.basePath,
                                self.arguments.plainTextOutputFile,
                                self.batch))

        if self.arguments.jsonOutputFile:
            self.reportManager.addOutput(
                JSONReport(requester.host, requester.port, requester.protocol,
                           requester.basePath, self.arguments.jsonOutputFile,
                           self.batch))

        if self.arguments.xmlOutputFile:
            self.reportManager.addOutput(
                XMLReport(requester.host, requester.port, requester.protocol,
                          requester.basePath, self.arguments.xmlOutputFile,
                          self.batch))

        if self.arguments.markdownOutputFile:
            self.reportManager.addOutput(
                MarkdownReport(requester.host, requester.port,
                               requester.protocol, requester.basePath,
                               self.arguments.markdownOutputFile, self.batch))
        if self.arguments.csvOutputFile:
            self.reportManager.addOutput(
                CSVReport(requester.host, requester.port, requester.protocol,
                          requester.basePath, self.arguments.csvOutputFile,
                          self.batch))

    # TODO: Refactor, this function should be a decorator for all the filters
    def matchCallback(self, path):
        self.index += 1

        if self.arguments.skip_on_429 and path.status == 429:
            self.skip429 = True
            return

        if (path.status and path.status not in self.excludeStatusCodes) and (
                not self.includeStatusCodes
                or path.status in self.includeStatusCodes
        ) and (not self.blacklists.get(path.status)
               or path.path not in self.blacklists.get(path.status)) and (
                   not self.excludeSizes
                   or FileUtils.size_human(len(path.response.body)).strip()
                   not in self.excludeSizes) and (
                       not self.minimumResponseSize
                       or self.minimumResponseSize < len(path.response.body)
                   ) and (not self.maximumResponseSize or
                          self.maximumResponseSize > len(path.response.body)):

            for excludeText in self.excludeTexts:
                if excludeText in path.response.body.decode('iso8859-1'):
                    del path
                    return

            for excludeRegexp in self.excludeRegexps:
                if (re.search(excludeRegexp,
                              path.response.body.decode('iso8859-1'))
                        is not None):
                    del path
                    return

            for excludeRedirect in self.excludeRedirects:
                if path.response.redirect and (
                    (re.match(excludeRedirect,
                              path.response.redirect.decode('iso8859-1'))
                     is not None) or
                    (excludeRedirect in path.response.redirect)):
                    del path
                    return

            addedToQueue = False

            if self.recursive and "?" not in path.path and "#" not in path.path:
                if path.response.redirect:
                    addedToQueue = self.addRedirectDirectory(path)
                else:
                    addedToQueue = self.addDirectory(path.path)

            self.output.statusReport(path.path, path.response,
                                     self.arguments.full_url, addedToQueue)

            if self.arguments.replay_proxy:
                self.requester.request(path.path,
                                       proxy=self.arguments.replay_proxy)

            newPath = self.currentDirectory + path.path

            self.reportManager.addPath(newPath, path.status, path.response)

            self.reportManager.save()

            del path

    def notFoundCallback(self, path):
        self.index += 1
        self.output.lastPath(path, self.index, len(self.dictionary),
                             self.currentJob, self.allJobs)
        del path

    def errorCallback(self, path, errorMsg):
        if self.arguments.exit_on_error:
            self.exit = True
            self.fuzzer.stop()
            self.output.error("\nCanceled due to an error")
            exit(1)

        else:
            if self.arguments.debug:
                self.output.debug(errorMsg)

            self.output.addConnectionError()

    def appendErrorLog(self, path, errorMsg):
        with self.threadsLock:
            line = time.strftime("[%y-%m-%d %H:%M:%S] - ")
            line += self.currentUrl + " - " + path + " - " + errorMsg
            self.errorLog.write(os.linesep + line)
            self.errorLog.flush()

    def handlePause(self, message):
        self.output.warning(message)
        self.fuzzer.pause()

        while self.fuzzer.stopped != len(self.fuzzer.threads):
            pass

        self.fuzzer.stopped = 0

        while True:
            msg = "[e]xit / [c]ontinue"

            if not self.directories.empty():
                msg += " / [n]ext"

            if len(self.urlList) > 1:
                msg += " / [s]kip target"

            self.output.inLine(msg + ": ")

            option = input()

            if option.lower() == "e":
                self.exit = True
                self.fuzzer.stop()
                self.output.error("\nCanceled by the user")
                exit(0)

            elif option.lower() == "c":
                self.fuzzer.resume()
                return

            elif option.lower() == "n" and not self.directories.empty():
                self.fuzzer.stop()
                return

            elif option.lower() == "s" and len(self.urlList) > 1:
                raise SkipTargetInterrupt

            else:
                continue

    def processPaths(self):
        while True:
            try:
                while not self.fuzzer.wait(0.25):
                    if self.skip429:
                        self.skip429 = False
                        self.fuzzer.pause()

                        while self.fuzzer.stopped != len(self.fuzzer.threads):
                            pass
                        self.output.error(
                            "\nSkipped the target due to 429 status code")

                        raise SkipTargetInterrupt

                break
            except (KeyboardInterrupt):
                self.handlePause(
                    "CTRL+C detected: Pausing threads, please wait...")

    def prepare(self):
        while not self.directories.empty():
            gc.collect()
            self.currentJob += 1
            self.index = 0
            self.currentDirectory = self.directories.get()
            self.output.warning("[{1}] Starting: {0}".format(
                self.currentDirectory, time.strftime("%H:%M:%S")))
            self.fuzzer.requester.basePath = self.output.basePath = self.basePath + self.currentDirectory
            self.fuzzer.start()
            self.processPaths()

        return

    def addPort(self, url):
        parsed = urllib.parse.urlparse(url)
        if ":" not in parsed.netloc:
            port = "443" if parsed.scheme == "https" else "80"
            url = url.replace(parsed.netloc, parsed.netloc + ":" + port)

        return url

    def addDirectory(self, path):
        if path.endswith("/"):
            if path in [directory + "/" for directory in self.excludeSubdirs]:
                return False

            dir = self.currentDirectory + path

            if self.scanSubdirs and dir in self.scanSubdirs:
                return False
            elif dir in self.doneDirs:
                return False
            elif self.recursive_depth and dir.count(
                    "/") > self.recursion_depth:
                return False

            self.directories.put(dir)
            self.allJobs += 1

            self.doneDirs.append(dir)

            return True

        else:
            return False

    def addRedirectDirectory(self, path):
        # Resolve the redirect header relative to the current URL and add the
        # path to self.directories if it is a subdirectory of the current URL

        baseUrl = self.currentUrl + self.currentDirectory
        baseUrl = self.addPort(baseUrl)

        absoluteUrl = urllib.parse.urljoin(baseUrl, path.response.redirect)
        absoluteUrl = self.addPort(absoluteUrl)

        if absoluteUrl.startswith(
                baseUrl) and absoluteUrl != baseUrl and absoluteUrl.endswith(
                    "/"):
            path = absoluteUrl[len(baseUrl):]

            if path in [directory + "/" for directory in self.excludeSubdirs]:
                return False

            dir = absoluteUrl[len(self.addPort(self.currentUrl)):]

            if self.scanSubdirs and dir in self.scanSubdirs:
                return False
            elif dir in self.doneDirs:
                return False
            elif self.recursion_depth and dir.count(
                    "/") > self.recursion_depth:
                return False

            self.directories.put(dir)
            self.allJobs += 1

            self.doneDirs.append(dir)

            return True

        return False
Пример #9
0
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = open(
            FileUtils.buildPath(script_path, "lib", "controller",
                                "banner.txt")).read().format(**VERSION)

        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.savePath = self.script_path
        self.doneDirs = []

        self.recursive_level_max = self.arguments.recursive_level_max

        if self.arguments.httpmethod.lower() not in ["get", "head", "post"]:
            self.output.error("Inavlid http method!")
            exit(1)

        self.httpmethod = self.arguments.httpmethod.lower()

        if self.arguments.saveHome:
            savePath = self.getSavePath()

            if not FileUtils.exists(savePath):
                FileUtils.createDirectory(savePath)

            if FileUtils.exists(savePath) and not FileUtils.isDir(savePath):
                self.output.error(
                    'Cannot use {} because is a file. Should be a directory'.
                    format(savePath))
                exit(1)

            if not FileUtils.canWrite(savePath):
                self.output.error(
                    'Directory {} is not writable'.format(savePath))
                exit(1)

            logs = FileUtils.buildPath(savePath, "logs")

            if not FileUtils.exists(logs):
                FileUtils.createDirectory(logs)

            reports = FileUtils.buildPath(savePath, "reports")

            if not FileUtils.exists(reports):
                FileUtils.createDirectory(reports)

            self.savePath = savePath

        self.reportsPath = FileUtils.buildPath(self.savePath, "logs")
        self.blacklists = self.getBlacklists()
        self.blacklists = {}
        self.fuzzer = None
        self.excludeStatusCodes = self.arguments.excludeStatusCodes
        self.excludeTexts = self.arguments.excludeTexts
        self.excludeRegexps = self.arguments.excludeRegexps
        self.recursive = self.arguments.recursive
        self.suppressEmpty = self.arguments.suppressEmpty
        self.directories = Queue()
        self.excludeSubdirs = (arguments.excludeSubdirs
                               if arguments.excludeSubdirs is not None else [])
        self.output.header(program_banner)
        # self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions,
        #                              self.arguments.lowercase, self.arguments.forceExtensions)
        # self.printConfig()
        self.errorLog = None
        self.errorLogPath = None
        self.errorLogLock = Lock()
        self.batch = False
        self.batchSession = None
        self.setupErrorLogs()
        self.output.newLine("\nError Log: {0}".format(self.errorLogPath))

        if self.arguments.autoSave and len(self.arguments.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(
                self.batchDirectoryPath))

        if self.arguments.useRandomAgents:
            self.randomAgents = FileUtils.getLines(
                FileUtils.buildPath(script_path, "db", "user-agents.txt"))

        try:
            for url in self.arguments.urlList:

                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url
                    self.output.target(self.currentUrl)

                    try:
                        # DNS A Record query
                        self.requester = Requester(
                            url,
                            script_path=self.script_path,
                            cookie=self.arguments.cookie,
                            useragent=self.arguments.useragent,
                            maxPool=self.arguments.threadsCount,
                            maxRetries=self.arguments.maxRetries,
                            delay=self.arguments.delay,
                            timeout=self.arguments.timeout,
                            ip=self.arguments.ip,
                            proxy=self.arguments.proxy,
                            redirect=self.arguments.redirect,
                            requestByHostname=self.arguments.requestByHostname,
                            httpmethod=self.httpmethod)
                        # 网站连通性测试
                        site_connection_test_resp = self.requester.request(
                            self.requester.basePath,
                            use_base_path=False,
                            allow_redirect=True,
                            fingerprint=True)
                        self.dictionary = Dictionary(self.requester.scan_list,
                                                     self.requester.directory,
                                                     self.requester.filename,
                                                     self.requester.extension)
                        # 404 page
                        if self.requester.url_type == URLType.normal_restful_dir:
                            path_404 = '{}/{}/'.format(
                                self.requester.basePath,
                                RandomUtils.randString(8))
                            path_404 = path_404.replace("//", "/")
                        elif self.requester.url_type == URLType.restful_file:
                            path_404 = self.requester.basePath.replace(
                                self.requester.filename,
                                RandomUtils.randString(
                                    len(self.requester.filename) or 8))
                        elif self.requester.url_type == URLType.normal_file:
                            path_404 = self.requester.basePath.replace(
                                self.requester.filename,
                                RandomUtils.randString(
                                    len(self.requester.filename) or 8))
                        path_404_quote = self.dictionary.quote(path_404)
                        response_404 = self.requester.request(
                            path_404_quote,
                            use_base_path=False,
                            allow_redirect=False)

                        # Waf 探测
                        waf_exist, waf_response = self.requester.waf_detect(
                            site_connection_test_resp.body,
                            url_quote=self.dictionary.quote)

                    except RequestException as e:
                        self.output.error(e.args[0]['message'])
                        raise SkipTargetInterrupt

                    if self.arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    for key, value in arguments.headers.items():
                        self.requester.setHeader(key, value)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath

                    if self.arguments.scanSubdirs is not None:
                        for subdir in self.arguments.scanSubdirs:
                            self.directories.put(subdir)

                    else:
                        self.directories.put('')

                    self.setupReports(self.requester)

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        waf_exist,
                        waf_response,
                        response_404,
                        testFailPath=self.arguments.testFailPath,
                        threads=self.arguments.threadsCount,
                        matchCallbacks=matchCallbacks,
                        notFoundCallbacks=notFoundCallbacks,
                        errorCallbacks=errorCallbacks)
                    try:
                        self.wait()
                    except RequestException as e:
                        self.output.error(
                            "Fatal error during site scanning: " +
                            e.args[0]['message'])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    continue

                finally:
                    self.reportManager.save()

        except KeyboardInterrupt:
            self.output.error('\nCanceled by the user')
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

            self.reportManager.close()

        self.output.warning('\nTask Completed')
Пример #10
0
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = (open(
            FileUtils.build_path(script_path, "lib", "controller",
                                 "banner.txt")).read().format(**VERSION))

        self.directories = Queue()
        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.done_dirs = []

        if arguments.raw_file:
            _raw = Raw(arguments.raw_file, arguments.scheme)
            self.url_list = [_raw.url()]
            self.httpmethod = _raw.method()
            self.data = _raw.data()
            self.headers = _raw.headers()
        else:
            default_headers = {
                "User-Agent":
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
                "Accept-Language": "*",
                "Accept-Encoding": "*",
                "Keep-Alive": "300",
                "Cache-Control": "max-age=0",
            }

            self.url_list = list(
                filter(None, dict.fromkeys(arguments.url_list)))
            self.httpmethod = arguments.httpmethod.lower()
            self.data = arguments.data
            self.headers = {**default_headers, **arguments.headers}
            if arguments.cookie:
                self.headers["Cookie"] = arguments.cookie
            if arguments.useragent:
                self.headers["User-Agent"] = arguments.useragent

        self.recursion_depth = arguments.recursion_depth

        if arguments.logs_location and self.validate_path(
                arguments.logs_location):
            self.logs_path = FileUtils.build_path(arguments.logs_location)
        elif self.validate_path(self.script_path):
            self.logs_path = FileUtils.build_path(self.script_path, "logs")
            if not FileUtils.exists(self.logs_path):
                FileUtils.create_directory(self.logs_path)

        if arguments.output_location and self.validate_path(
                arguments.output_location):
            self.save_path = FileUtils.build_path(arguments.output_location)
        elif self.validate_path(self.script_path):
            self.save_path = FileUtils.build_path(self.script_path, "reports")
            if not FileUtils.exists(self.save_path):
                FileUtils.create_directory(self.save_path)

        self.blacklists = self.get_blacklists()
        self.include_status_codes = arguments.include_status_codes
        self.exclude_status_codes = arguments.exclude_status_codes
        self.exclude_sizes = arguments.exclude_sizes
        self.exclude_texts = arguments.exclude_texts
        self.exclude_regexps = arguments.exclude_regexps
        self.exclude_redirects = arguments.exclude_redirects
        self.recursive = arguments.recursive
        self.deep_recursive = arguments.deep_recursive
        self.force_recursive = arguments.force_recursive
        self.recursion_status_codes = arguments.recursion_status_codes
        self.minimum_response_size = arguments.minimum_response_size
        self.maximum_response_size = arguments.maximum_response_size
        self.maxtime = arguments.maxtime
        self.scan_subdirs = arguments.scan_subdirs
        self.exclude_subdirs = arguments.exclude_subdirs

        self.dictionary = Dictionary(
            paths=arguments.wordlist,
            extensions=arguments.extensions,
            suffixes=arguments.suffixes,
            prefixes=arguments.prefixes,
            lowercase=arguments.lowercase,
            uppercase=arguments.uppercase,
            capitalization=arguments.capitalization,
            forced_extensions=arguments.force_extensions,
            exclude_extensions=arguments.exclude_extensions,
            no_extension=arguments.no_extension,
            only_selected=arguments.only_selected)

        self.all_jobs = len(self.scan_subdirs) if self.scan_subdirs else 1
        self.current_job = 0
        self.start_time = time.time()
        self.error_log = None
        self.error_log_path = None
        self.threads_lock = Lock()
        self.batch = False
        self.batch_session = None

        self.output.header(program_banner)
        self.print_config()

        if arguments.use_random_agents:
            self.random_agents = FileUtils.get_lines(
                FileUtils.build_path(script_path, "db", "user-agents.txt"))

        self.report_manager = EmptyReportManager()
        self.report = EmptyReport()
        if arguments.autosave_report or arguments.output_file:
            if len(self.url_list) > 1:
                self.setup_batch_reports()
            self.setup_reports()

        self.setup_error_logs()
        self.output.error_log_file(self.error_log_path)
        try:
            for url in self.url_list:
                try:
                    gc.collect()
                    url = url if url.endswith("/") else url + "/"
                    self.output.set_target(url, self.arguments.scheme)

                    try:
                        self.requester = Requester(
                            url,
                            max_pool=arguments.threads_count,
                            max_retries=arguments.max_retries,
                            timeout=arguments.timeout,
                            ip=arguments.ip,
                            proxy=arguments.proxy,
                            proxylist=arguments.proxylist,
                            redirect=arguments.redirect,
                            request_by_hostname=arguments.request_by_hostname,
                            httpmethod=self.httpmethod,
                            data=self.data,
                            scheme=arguments.scheme,
                        )

                        for key, value in self.headers.items():
                            self.requester.set_header(key, value)

                        if arguments.auth:
                            self.requester.set_auth(arguments.auth_type,
                                                    arguments.auth)

                        self.requester.request("")

                        if arguments.autosave_report or arguments.output_file:
                            self.report = Report(self.requester.host,
                                                 self.requester.port,
                                                 self.requester.protocol,
                                                 self.requester.base_path)

                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                    if arguments.use_random_agents:
                        self.requester.set_random_agents(self.random_agents)

                    # Initialize directories Queue with start Path
                    self.base_path = self.requester.base_path
                    self.status_skip = None

                    for subdir in self.scan_subdirs:
                        self.directories.put(subdir)

                    else:
                        self.directories.put("")

                    match_callbacks = [self.match_callback]
                    not_found_callbacks = [self.not_found_callback]
                    error_callbacks = [
                        self.error_callback, self.append_error_log
                    ]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        suffixes=arguments.suffixes,
                        prefixes=arguments.prefixes,
                        exclude_content=arguments.exclude_content,
                        threads=arguments.threads_count,
                        delay=arguments.delay,
                        maxrate=arguments.maxrate,
                        match_callbacks=match_callbacks,
                        not_found_callbacks=not_found_callbacks,
                        error_callbacks=error_callbacks,
                    )
                    try:
                        self.prepare()
                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    self.report.completed = True
                    continue

        except KeyboardInterrupt:
            self.output.error("\nCanceled by the user")
            exit(0)

        finally:
            if not self.error_log.closed:
                self.error_log.close()

        self.output.warning("\nTask Completed")
Пример #11
0
class Controller(object):
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = (open(
            FileUtils.build_path(script_path, "lib", "controller",
                                 "banner.txt")).read().format(**VERSION))

        self.directories = Queue()
        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.done_dirs = []

        if arguments.raw_file:
            _raw = Raw(arguments.raw_file, arguments.scheme)
            self.url_list = [_raw.url()]
            self.httpmethod = _raw.method()
            self.data = _raw.data()
            self.headers = _raw.headers()
        else:
            default_headers = {
                "User-Agent":
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
                "Accept-Language": "*",
                "Accept-Encoding": "*",
                "Keep-Alive": "300",
                "Cache-Control": "max-age=0",
            }

            self.url_list = list(
                filter(None, dict.fromkeys(arguments.url_list)))
            self.httpmethod = arguments.httpmethod.lower()
            self.data = arguments.data
            self.headers = {**default_headers, **arguments.headers}
            if arguments.cookie:
                self.headers["Cookie"] = arguments.cookie
            if arguments.useragent:
                self.headers["User-Agent"] = arguments.useragent

        self.recursion_depth = arguments.recursion_depth

        if arguments.logs_location and self.validate_path(
                arguments.logs_location):
            self.logs_path = FileUtils.build_path(arguments.logs_location)
        elif self.validate_path(self.script_path):
            self.logs_path = FileUtils.build_path(self.script_path, "logs")
            if not FileUtils.exists(self.logs_path):
                FileUtils.create_directory(self.logs_path)

        if arguments.output_location and self.validate_path(
                arguments.output_location):
            self.save_path = FileUtils.build_path(arguments.output_location)
        elif self.validate_path(self.script_path):
            self.save_path = FileUtils.build_path(self.script_path, "reports")
            if not FileUtils.exists(self.save_path):
                FileUtils.create_directory(self.save_path)

        self.blacklists = self.get_blacklists()
        self.include_status_codes = arguments.include_status_codes
        self.exclude_status_codes = arguments.exclude_status_codes
        self.exclude_sizes = arguments.exclude_sizes
        self.exclude_texts = arguments.exclude_texts
        self.exclude_regexps = arguments.exclude_regexps
        self.exclude_redirects = arguments.exclude_redirects
        self.recursive = arguments.recursive
        self.deep_recursive = arguments.deep_recursive
        self.force_recursive = arguments.force_recursive
        self.recursion_status_codes = arguments.recursion_status_codes
        self.minimum_response_size = arguments.minimum_response_size
        self.maximum_response_size = arguments.maximum_response_size
        self.maxtime = arguments.maxtime
        self.scan_subdirs = arguments.scan_subdirs
        self.exclude_subdirs = arguments.exclude_subdirs

        self.dictionary = Dictionary(
            paths=arguments.wordlist,
            extensions=arguments.extensions,
            suffixes=arguments.suffixes,
            prefixes=arguments.prefixes,
            lowercase=arguments.lowercase,
            uppercase=arguments.uppercase,
            capitalization=arguments.capitalization,
            forced_extensions=arguments.force_extensions,
            exclude_extensions=arguments.exclude_extensions,
            no_extension=arguments.no_extension,
            only_selected=arguments.only_selected)

        self.all_jobs = len(self.scan_subdirs) if self.scan_subdirs else 1
        self.current_job = 0
        self.start_time = time.time()
        self.error_log = None
        self.error_log_path = None
        self.threads_lock = Lock()
        self.batch = False
        self.batch_session = None

        self.output.header(program_banner)
        self.print_config()

        if arguments.use_random_agents:
            self.random_agents = FileUtils.get_lines(
                FileUtils.build_path(script_path, "db", "user-agents.txt"))

        self.report_manager = EmptyReportManager()
        self.report = EmptyReport()
        if arguments.autosave_report or arguments.output_file:
            if len(self.url_list) > 1:
                self.setup_batch_reports()
            self.setup_reports()

        self.setup_error_logs()
        self.output.error_log_file(self.error_log_path)
        try:
            for url in self.url_list:
                try:
                    gc.collect()
                    url = url if url.endswith("/") else url + "/"
                    self.output.set_target(url, self.arguments.scheme)

                    try:
                        self.requester = Requester(
                            url,
                            max_pool=arguments.threads_count,
                            max_retries=arguments.max_retries,
                            timeout=arguments.timeout,
                            ip=arguments.ip,
                            proxy=arguments.proxy,
                            proxylist=arguments.proxylist,
                            redirect=arguments.redirect,
                            request_by_hostname=arguments.request_by_hostname,
                            httpmethod=self.httpmethod,
                            data=self.data,
                            scheme=arguments.scheme,
                        )

                        for key, value in self.headers.items():
                            self.requester.set_header(key, value)

                        if arguments.auth:
                            self.requester.set_auth(arguments.auth_type,
                                                    arguments.auth)

                        self.requester.request("")

                        if arguments.autosave_report or arguments.output_file:
                            self.report = Report(self.requester.host,
                                                 self.requester.port,
                                                 self.requester.protocol,
                                                 self.requester.base_path)

                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                    if arguments.use_random_agents:
                        self.requester.set_random_agents(self.random_agents)

                    # Initialize directories Queue with start Path
                    self.base_path = self.requester.base_path
                    self.status_skip = None

                    for subdir in self.scan_subdirs:
                        self.directories.put(subdir)

                    else:
                        self.directories.put("")

                    match_callbacks = [self.match_callback]
                    not_found_callbacks = [self.not_found_callback]
                    error_callbacks = [
                        self.error_callback, self.append_error_log
                    ]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        suffixes=arguments.suffixes,
                        prefixes=arguments.prefixes,
                        exclude_content=arguments.exclude_content,
                        threads=arguments.threads_count,
                        delay=arguments.delay,
                        maxrate=arguments.maxrate,
                        match_callbacks=match_callbacks,
                        not_found_callbacks=not_found_callbacks,
                        error_callbacks=error_callbacks,
                    )
                    try:
                        self.prepare()
                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    self.report.completed = True
                    continue

        except KeyboardInterrupt:
            self.output.error("\nCanceled by the user")
            exit(0)

        finally:
            if not self.error_log.closed:
                self.error_log.close()

        self.output.warning("\nTask Completed")

    # Print dirsearch metadata (threads, HTTP method, ...)
    def print_config(self):
        self.output.config(
            ', '.join(self.arguments.extensions),
            ', '.join(self.arguments.prefixes),
            ', '.join(self.arguments.suffixes),
            str(self.arguments.threads_count),
            str(len(self.dictionary)),
            str(self.httpmethod),
        )

    # Check if the runtime exceeded the maximum runtime set by user
    def is_timed_out(self):
        if self.maxtime and time.time() - self.start_time > self.maxtime:
            return True

        return False

    # Get ignore paths for status codes
    # TODO: Move this to /lib/core/dictionary.py
    def get_blacklists(self):
        reext = re.compile(r'\%ext\%', re.IGNORECASE)
        blacklists = {}

        for status in [400, 403, 500]:
            blacklist_file_name = FileUtils.build_path(self.script_path, "db")
            blacklist_file_name = FileUtils.build_path(
                blacklist_file_name, "{}_blacklist.txt".format(status))

            if not FileUtils.can_read(blacklist_file_name):
                # Skip if cannot read file
                continue

            blacklists[status] = []

            for line in FileUtils.get_lines(blacklist_file_name):
                # Skip comments
                if line.lstrip().startswith("#"):
                    continue

                if line.startswith("/"):
                    line = line[1:]

                # Classic dirsearch blacklist processing (with %EXT% keyword)
                if "%ext%" in line.lower():
                    for extension in self.arguments.extensions:
                        entry = reext.sub(extension, line)

                        blacklists[status].append(entry)

                # Forced extensions is not used here because -r is only used for wordlist,
                # applying in blacklist may create false negatives

                else:
                    blacklists[status].append(line)

        return blacklists

    # Create error log file
    def setup_error_logs(self):
        file_name = "errors-{0}.log".format(time.strftime("%y-%m-%d_%H-%M-%S"))
        self.error_log_path = FileUtils.build_path(self.logs_path, file_name)

        try:
            self.error_log = open(self.error_log_path, "w")
        except PermissionError:
            self.output.error(
                "Couldn't create the error log. Try running again with highest permission"
            )
            sys.exit(1)

    # Create batch report folder
    def setup_batch_reports(self):
        self.batch = True
        if not self.arguments.output_file:
            self.batch_session = "BATCH-{0}".format(
                time.strftime("%y-%m-%d_%H-%M-%S"))
            self.batch_directory_path = FileUtils.build_path(
                self.save_path, self.batch_session)

            if not FileUtils.exists(self.batch_directory_path):
                FileUtils.create_directory(self.batch_directory_path)

                if not FileUtils.exists(self.batch_directory_path):
                    self.output.error(
                        "Couldn't create batch folder at {}".format(
                            self.batch_directory_path))
                    sys.exit(1)

    # Get file extension for report format
    def get_output_extension(self):
        if self.arguments.output_format and self.arguments.output_format not in [
                "plain", "simple"
        ]:
            return ".{0}".format(self.arguments.output_format)
        else:
            return ".txt"

    # Create report file
    def setup_reports(self):
        if self.arguments.output_file:
            output_file = FileUtils.get_abs_path(self.arguments.output_file)
            self.output.output_file(output_file)
        else:
            if self.batch:
                file_name = "BATCH"
                file_name += self.get_output_extension()
                directory_path = self.batch_directory_path
            else:
                local_requester = Requester(self.url_list[0])
                file_name = ("{}_".format(
                    local_requester.base_path.replace(os.path.sep, ".")[:-1]))
                file_name += time.strftime("%y-%m-%d_%H-%M-%S")
                file_name += self.get_output_extension()
                directory_path = FileUtils.build_path(self.save_path,
                                                      local_requester.host)

            output_file = FileUtils.build_path(directory_path, file_name)

            if FileUtils.exists(output_file):
                i = 2

                while FileUtils.exists(output_file + "_" + str(i)):
                    i += 1

                output_file += "_" + str(i)

            if not FileUtils.exists(directory_path):
                FileUtils.create_directory(directory_path)

                if not FileUtils.exists(directory_path):
                    self.output.error(
                        "Couldn't create the reports folder at {}".format(
                            directory_path))
                    sys.exit(1)

            self.output.output_file(output_file)

        if self.arguments.output_file and self.arguments.output_format:
            self.report_manager = ReportManager(self.arguments.output_format,
                                                self.arguments.output_file)
        elif self.arguments.output_format:
            self.report_manager = ReportManager(self.arguments.output_format,
                                                output_file)
        else:
            self.report_manager = ReportManager("plain", output_file)

    # Check if given path is valid (can read/write)
    def validate_path(self, path):
        if not FileUtils.exists(path):
            self.output.error("{} does not exist".format(path))
            exit(1)

        if FileUtils.exists(path) and not FileUtils.is_dir(path):
            self.output.error(
                "{} is a file, should be a directory".format(path))
            exit(1)

        if not FileUtils.can_write(path):
            self.output.error("Directory {} is not writable".format(path))
            exit(1)

        return True

    # Validate the response by different filters
    def valid(self, path):
        if not path:
            return False

        if path.status in self.exclude_status_codes:
            return False

        if self.include_status_codes and path.status not in self.include_status_codes:
            return False

        if self.blacklists.get(
                path.status) and path.path in self.blacklists.get(path.status):
            return False

        if self.exclude_sizes and FileUtils.size_human(len(
                path.response.body)).strip() in self.exclude_sizes:
            return False

        if self.minimum_response_size and self.minimum_response_size > len(
                path.response.body):
            return False

        if self.maximum_response_size and self.maximum_response_size < len(
                path.response.body):
            return False

        for exclude_text in self.exclude_texts:
            if exclude_text in path.response.body.decode('iso8859-1'):
                return False

        for exclude_regexp in self.exclude_regexps:
            if (re.search(exclude_regexp,
                          path.response.body.decode('iso8859-1')) is not None):
                return False

        for exclude_redirect in self.exclude_redirects:
            if path.response.redirect and (
                (re.match(exclude_redirect, path.response.redirect)
                 is not None) or (exclude_redirect in path.response.redirect)):
                return False

        return True

    # Callback for found paths
    def match_callback(self, path):
        self.index += 1

        for status in self.arguments.skip_on_status:
            if path.status == status:
                self.status_skip = status
                return

        if not self.valid(path):
            del path
            return

        added_to_queue = False

        if (any([self.recursive, self.deep_recursive, self.force_recursive
                 ])) and (not self.recursion_status_codes
                          or path.status in self.recursion_status_codes):
            if path.response.redirect:
                added_to_queue = self.add_redirect_directory(path)
            else:
                added_to_queue = self.add_directory(path.path)

        self.output.status_report(path.path, path.response,
                                  self.arguments.full_url, added_to_queue)

        if self.arguments.replay_proxy:
            self.requester.request(path.path,
                                   proxy=self.arguments.replay_proxy)

        new_path = self.current_directory + path.path

        self.report.add_result(new_path, path.status, path.response)
        self.report_manager.update_report(self.report)

        del path

    # Callback for invalid paths
    def not_found_callback(self, path):
        self.index += 1
        self.output.last_path(path, self.index, len(self.dictionary),
                              self.current_job, self.all_jobs,
                              self.fuzzer.rate)
        del path

    # Callback for errors while fuzzing
    def error_callback(self, path, error_msg):
        if self.arguments.exit_on_error:
            self.exit = True
            self.fuzzer.stop()
            self.output.error("\nCanceled due to an error")
            exit(1)

        else:
            self.output.add_connection_error()

    # Write error to log file
    def append_error_log(self, path, error_msg):
        with self.threads_lock:
            line = time.strftime("[%y-%m-%d %H:%M:%S] - ")
            line += self.requester.base_url + " - " + self.base_path + self.current_directory + path + " - " + error_msg
            self.error_log.write(os.linesep + line)
            self.error_log.flush()

    # Handle CTRL+C
    def handle_pause(self, message):
        self.output.warning(message)
        self.fuzzer.pause()

        # If one of the tasks is broken, don't let the user wait forever
        for i in range(300):
            if self.fuzzer.stopped == len(self.fuzzer.threads):
                break
            time.sleep(0.025)

        self.fuzzer.stopped = 0

        while True:
            msg = "[q]uit / [c]ontinue"

            if not self.directories.empty():
                msg += " / [n]ext"

            if len(self.url_list) > 1:
                msg += " / [s]kip target"

            self.output.in_line(msg + ": ")

            option = input()

            if option.lower() == "q":
                self.exit = True
                self.fuzzer.stop()
                self.output.error("\nCanceled by the user")
                self.report_manager.update_report(self.report)
                exit(0)

            elif option.lower() == "c":
                self.fuzzer.resume()
                return

            elif option.lower() == "n" and not self.directories.empty():
                self.fuzzer.stop()
                return

            elif option.lower() == "s" and len(self.url_list) > 1:
                self.output.new_line()
                raise SkipTargetInterrupt

    # Monitor the fuzzing process
    def process_paths(self):
        while True:
            try:
                while not self.fuzzer.wait(0.25):
                    # Check if the "skip status code" was returned
                    if self.status_skip:
                        self.fuzzer.pause()
                        time.sleep(1.5)

                        self.output.error(
                            "\nSkipped the target due to {0} status code".
                            format(self.status_skip))

                        raise SkipTargetInterrupt

                    elif self.is_timed_out():
                        self.output.error(
                            "\nCanceled because the runtime exceeded the maximal set by user"
                        )
                        exit(0)

                break

            except KeyboardInterrupt:
                self.handle_pause(
                    "CTRL+C detected: Pausing threads, please wait...")

    # Preparation between subdirectory scans
    def prepare(self):
        while not self.directories.empty():
            gc.collect()
            self.current_job += 1
            self.index = 0
            self.current_directory = self.directories.get()
            self.output.warning("[{1}] Starting: {0}".format(
                self.current_directory, time.strftime("%H:%M:%S")))
            self.fuzzer.requester.base_path = self.output.base_path = self.base_path + self.current_directory
            self.fuzzer.start()
            self.process_paths()

        self.report.completed = True
        self.report_manager.update_report(self.report)
        self.report = None

        return

    # Add directory to the recursion queue
    def add_directory(self, path):
        added = False
        path = path.split("?")[0].split("#")[0]

        if any(
            [path.startswith(directory)
             for directory in self.exclude_subdirs]):
            return False

        full_path = self.current_directory + path

        dirs = []

        if self.deep_recursive:
            for i in range(1, path.count("/") + 1):
                dir = full_path.replace(path, "") + "/".join(
                    path.split("/")[:i])
                dirs.append(dir.rstrip("/") + "/")
        if self.force_recursive:
            if not full_path.endswith("/"):
                full_path += "/"
            dirs.append(full_path)
        elif self.recursive and full_path.endswith("/"):
            dirs.append(full_path)

        for dir in dirs:
            if dir in self.scan_subdirs:
                continue
            elif dir in self.done_dirs:
                continue
            elif self.recursion_depth and dir.count(
                    "/") > self.recursion_depth:
                continue

            self.directories.put(dir)
            self.done_dirs.append(dir)

            self.all_jobs += 1
            added = True

        return added

    # Add port to the URL
    def add_port(self, url):
        chunks = url.split("/")
        if ":" not in chunks[2]:
            chunks[2] += (":80" if chunks[0] == "http:" else ":443")
            url = "/".join(chunks)

        return url

    # Resolve the redirect and add the path to the recursion queue
    # if it's a subdirectory of the current URL
    def add_redirect_directory(self, path):
        base_url = self.requester.base_url + self.base_path + self.current_directory + path.path

        redirect_url = urljoin(self.requester.base_url, path.response.redirect)
        redirect_url = self.add_port(redirect_url)

        if redirect_url.startswith(base_url + "/"):
            path = redirect_url[len(self.requester.base_url + self.base_path +
                                    self.current_directory):]

            return self.add_directory(path)

        return False
Пример #12
0
class Controller(object):
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = (open(
            FileUtils.build_path(script_path, "lib", "controller",
                                 "banner.txt")).read().format(**VERSION))

        self.directories = Queue()
        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.doneDirs = []

        if arguments.raw_file:
            _raw = Raw(arguments.raw_file, arguments.scheme)
            self.urlList = [_raw.url()]
            self.httpmethod = _raw.method()
            self.data = _raw.data()
            self.headers = _raw.headers()
        else:
            default_headers = {
                "User-Agent":
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
                "Accept-Language": "*",
                "Accept-Encoding": "*",
                "Keep-Alive": "300",
                "Cache-Control": "max-age=0",
            }

            self.urlList = list(filter(None, dict.fromkeys(arguments.urlList)))
            self.httpmethod = arguments.httpmethod.lower()
            self.data = arguments.data
            self.headers = {**default_headers, **arguments.headers}
            if arguments.cookie:
                self.headers["Cookie"] = arguments.cookie
            if arguments.cookie:
                self.headers["User-Agent"] = arguments.useragent

        self.recursion_depth = arguments.recursion_depth

        if arguments.logsLocation and self.validatePath(
                arguments.logsLocation):
            self.logsPath = FileUtils.build_path(arguments.logsLocation)
        else:
            self.logsPath = FileUtils.build_path(self.script_path, "logs")
            if not FileUtils.exists(self.logsPath):
                FileUtils.create_directory(self.logsPath)

        if arguments.outputLocation and self.validatePath(
                arguments.outputLocation):
            self.save_path = FileUtils.build_path(arguments.outputLocation)
        else:
            self.save_path = FileUtils.build_path(self.script_path, "reports")
            if not FileUtils.exists(self.save_path):
                FileUtils.create_directory(self.save_path)

        self.blacklists = self.getBlacklists()
        self.includeStatusCodes = arguments.includeStatusCodes
        self.excludeStatusCodes = arguments.excludeStatusCodes
        self.excludeSizes = arguments.excludeSizes
        self.excludeTexts = arguments.excludeTexts
        self.excludeRegexps = arguments.excludeRegexps
        self.excludeRedirects = arguments.excludeRedirects
        self.recursive = arguments.recursive
        self.deep_recursive = arguments.deep_recursive
        self.force_recursive = arguments.force_recursive
        self.recursionStatusCodes = arguments.recursionStatusCodes
        self.minimumResponseSize = arguments.minimumResponseSize
        self.maximumResponseSize = arguments.maximumResponseSize
        self.maxtime = arguments.maxtime
        self.scanSubdirs = arguments.scanSubdirs
        self.excludeSubdirs = (arguments.excludeSubdirs
                               if arguments.excludeSubdirs else [])

        self.dictionary = Dictionary(
            paths=arguments.wordlist,
            extensions=arguments.extensions,
            suffixes=arguments.suffixes,
            prefixes=arguments.prefixes,
            lowercase=arguments.lowercase,
            uppercase=arguments.uppercase,
            capitalization=arguments.capitalization,
            forcedExtensions=arguments.forceExtensions,
            excludeExtensions=arguments.excludeExtensions,
            noExtension=arguments.noExtension,
            onlySelected=arguments.onlySelected)

        self.allJobs = len(self.scanSubdirs) if self.scanSubdirs else 1
        self.currentJob = 0
        self.startTime = time.time()
        self.errorLog = None
        self.errorLogPath = None
        self.threadsLock = Lock()
        self.batch = False
        self.batchSession = None

        self.output.header(program_banner)
        self.printConfig()

        if arguments.useRandomAgents:
            self.randomAgents = FileUtils.get_lines(
                FileUtils.build_path(script_path, "db", "user-agents.txt"))

        self.reportManager = EmptyReportManager()
        self.report = EmptyReport()
        if arguments.autosaveReport or arguments.outputFile:
            if len(self.urlList) > 1:
                self.setupBatchReports()
            self.setupReports()

        self.setupErrorLogs()
        self.output.errorLogFile(self.errorLogPath)
        try:
            for url in self.urlList:
                try:
                    gc.collect()
                    self.currentUrl = url if url.endswith("/") else url + "/"
                    self.output.setTarget(self.currentUrl,
                                          self.arguments.scheme)

                    try:
                        self.requester = Requester(
                            url,
                            maxPool=arguments.threadsCount,
                            maxRetries=arguments.maxRetries,
                            timeout=arguments.timeout,
                            ip=arguments.ip,
                            proxy=arguments.proxy,
                            proxylist=arguments.proxylist,
                            redirect=arguments.redirect,
                            requestByHostname=arguments.requestByHostname,
                            httpmethod=self.httpmethod,
                            data=self.data,
                            scheme=arguments.scheme,
                        )

                        if arguments.autosaveReport or arguments.outputFile:
                            self.report = Report(self.requester.host,
                                                 self.requester.port,
                                                 self.requester.protocol,
                                                 self.requester.basePath)

                        for key, value in self.headers.items():
                            self.requester.setHeader(key, value)

                        if arguments.auth:
                            self.requester.setAuth(arguments.auth_type,
                                                   arguments.auth)

                        self.requester.request("")

                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                    if arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath
                    self.status_skip = None

                    if self.scanSubdirs:
                        for subdir in self.scanSubdirs:
                            self.directories.put(subdir)

                    else:
                        self.directories.put("")

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        suffixes=arguments.suffixes,
                        prefixes=arguments.prefixes,
                        excludeContent=arguments.excludeContent,
                        threads=arguments.threadsCount,
                        delay=arguments.delay,
                        maxrate=arguments.maxrate,
                        matchCallbacks=matchCallbacks,
                        notFoundCallbacks=notFoundCallbacks,
                        errorCallbacks=errorCallbacks,
                    )
                    try:
                        self.prepare()
                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    self.report.completed = True
                    continue

        except KeyboardInterrupt:
            self.output.error("\nCanceled by the user")
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

        self.output.warning("\nTask Completed")

    def printConfig(self):
        self.output.config(
            ', '.join(self.arguments.extensions),
            ', '.join(self.arguments.prefixes),
            ', '.join(self.arguments.suffixes),
            str(self.arguments.threadsCount),
            str(len(self.dictionary)),
            str(self.httpmethod),
        )

    def getBlacklists(self):
        reext = re.compile(r'\%ext\%', re.IGNORECASE)
        blacklists = {}

        for status in [400, 403, 500]:
            blacklistFileName = FileUtils.build_path(self.script_path, "db")
            blacklistFileName = FileUtils.build_path(
                blacklistFileName, "{}_blacklist.txt".format(status))

            if not FileUtils.can_read(blacklistFileName):
                # Skip if cannot read file
                continue

            blacklists[status] = []

            for line in FileUtils.get_lines(blacklistFileName):
                # Skip comments
                if line.lstrip().startswith("#"):
                    continue

                if line.startswith("/"):
                    line = line[1:]

                # Classic dirsearch blacklist processing (with %EXT% keyword)
                if "%ext%" in line.lower():
                    for extension in self.arguments.extensions:
                        entry = reext.sub(extension, line)

                        blacklists[status].append(entry)

                # Forced extensions is not used here because -r is only used for wordlist,
                # applying in blacklist may create false negatives

                else:
                    blacklists[status].append(line)

        return blacklists

    def setupErrorLogs(self):
        fileName = "errors-{0}.log".format(time.strftime("%y-%m-%d_%H-%M-%S"))
        self.errorLogPath = FileUtils.build_path(self.logsPath, fileName)

        try:
            self.errorLog = open(self.errorLogPath, "w")
        except PermissionError:
            self.output.error(
                "Couldn't create the error log. Try running again with highest permission"
            )
            sys.exit(1)

    def setupBatchReports(self):
        self.batch = True
        if not self.arguments.outputFile:
            self.batchSession = "BATCH-{0}".format(
                time.strftime("%y-%m-%d_%H-%M-%S"))
            self.batchDirectoryPath = FileUtils.build_path(
                self.save_path, self.batchSession)

            if not FileUtils.exists(self.batchDirectoryPath):
                FileUtils.create_directory(self.batchDirectoryPath)

                if not FileUtils.exists(self.batchDirectoryPath):
                    self.output.error(
                        "Couldn't create batch folder at {}".format(
                            self.batchDirectoryPath))
                    sys.exit(1)

    def getOutputExtension(self):
        if self.arguments.outputFormat and self.arguments.outputFormat not in [
                "plain", "simple"
        ]:
            return ".{0}".format(self.arguments.outputFormat)
        else:
            return ".txt"

    def setupReports(self):
        if self.arguments.outputFile:
            outputFile = FileUtils.get_abs_path(self.arguments.outputFile)
            self.output.outputFile(outputFile)
        else:
            if self.batch:
                fileName = "BATCH"
                fileName += self.getOutputExtension()
                directoryPath = self.batchDirectoryPath
            else:
                localRequester = Requester(self.urlList[0])
                fileName = ("{}_".format(
                    localRequester.basePath.replace(os.path.sep, ".")[:-1]))
                fileName += time.strftime("%y-%m-%d_%H-%M-%S")
                fileName += self.getOutputExtension()
                directoryPath = FileUtils.build_path(self.save_path,
                                                     localRequester.host)

            outputFile = FileUtils.build_path(directoryPath, fileName)

            if FileUtils.exists(outputFile):
                i = 2

                while FileUtils.exists(outputFile + "_" + str(i)):
                    i += 1

                outputFile += "_" + str(i)

            if not FileUtils.exists(directoryPath):
                FileUtils.create_directory(directoryPath)

                if not FileUtils.exists(directoryPath):
                    self.output.error(
                        "Couldn't create the reports folder at {}".format(
                            directoryPath))
                    sys.exit(1)

            self.output.outputFile(outputFile)

        if self.arguments.outputFile and self.arguments.outputFormat:
            self.reportManager = ReportManager(self.arguments.outputFormat,
                                               self.arguments.outputFile)
        elif self.arguments.outputFormat:
            self.reportManager = ReportManager(self.arguments.outputFormat,
                                               outputFile)
        else:
            self.reportManager = ReportManager("plain", outputFile)

    def validatePath(self, path):
        if not FileUtils.exists(path):
            self.output.error("{} does not exist".format(path))
            exit(1)

        if FileUtils.exists(path) and not FileUtils.is_dir(path):
            self.output.error(
                "{} is a file, should be a directory".format(path))
            exit(1)

        if not FileUtils.can_write(path):
            self.output.error("Directory {} is not writable".format(path))
            exit(1)

        return True

    # TODO: Refactor, this function should be a decorator for all the filters
    def matchCallback(self, path):
        self.index += 1

        for status in self.arguments.skip_on_status:
            if path.status == status:
                self.status_skip = status
                return

        if (path.status and path.status not in self.excludeStatusCodes) and (
                not self.includeStatusCodes
                or path.status in self.includeStatusCodes
        ) and (not self.blacklists.get(path.status)
               or path.path not in self.blacklists.get(path.status)) and (
                   not self.excludeSizes
                   or FileUtils.size_human(len(path.response.body)).strip()
                   not in self.excludeSizes) and (
                       not self.minimumResponseSize
                       or self.minimumResponseSize < len(path.response.body)
                   ) and (not self.maximumResponseSize or
                          self.maximumResponseSize > len(path.response.body)):

            for excludeText in self.excludeTexts:
                if excludeText in path.response.body.decode('iso8859-1'):
                    del path
                    return

            for excludeRegexp in self.excludeRegexps:
                if (re.search(excludeRegexp,
                              path.response.body.decode('iso8859-1'))
                        is not None):
                    del path
                    return

            for excludeRedirect in self.excludeRedirects:
                if path.response.redirect and (
                    (re.match(excludeRedirect,
                              path.response.redirect.decode('iso8859-1'))
                     is not None) or
                    (excludeRedirect in path.response.redirect)):
                    del path
                    return

            addedToQueue = False

            if (any([
                    self.recursive, self.deep_recursive, self.force_recursive
            ])) and (not self.recursionStatusCodes
                     or path.status in self.recursionStatusCodes):
                if path.response.redirect:
                    addedToQueue = self.addRedirectDirectory(path)
                else:
                    addedToQueue = self.addDirectory(path.path)

            self.output.statusReport(path.path, path.response,
                                     self.arguments.full_url, addedToQueue)

            if self.arguments.replay_proxy:
                self.requester.request(path.path,
                                       proxy=self.arguments.replay_proxy)

            newPath = self.currentDirectory + path.path

            self.report.addResult(newPath, path.status, path.response)
            self.reportManager.updateReport(self.report)

            del path

    def notFoundCallback(self, path):
        self.index += 1
        self.output.lastPath(path, self.index, len(self.dictionary),
                             self.currentJob, self.allJobs, self.fuzzer.rate)
        del path

    def errorCallback(self, path, errorMsg):
        if self.arguments.exit_on_error:
            self.exit = True
            self.fuzzer.stop()
            self.output.error("\nCanceled due to an error")
            exit(1)

        else:
            self.output.addConnectionError()

    def appendErrorLog(self, path, errorMsg):
        with self.threadsLock:
            line = time.strftime("[%y-%m-%d %H:%M:%S] - ")
            line += self.currentUrl + " - " + path + " - " + errorMsg
            self.errorLog.write(os.linesep + line)
            self.errorLog.flush()

    def handlePause(self, message):
        self.output.warning(message)
        self.fuzzer.pause()

        # If one of the tasks is broken, don't let the user wait forever
        for i in range(300):
            if self.fuzzer.stopped == len(self.fuzzer.threads):
                break
            time.sleep(0.025)

        self.fuzzer.stopped = 0

        while True:
            msg = "[q]uit / [c]ontinue"

            if not self.directories.empty():
                msg += " / [n]ext"

            if len(self.urlList) > 1:
                msg += " / [s]kip target"

            self.output.inLine(msg + ": ")

            option = input()

            if option.lower() == "q":
                self.exit = True
                self.fuzzer.stop()
                self.output.error("\nCanceled by the user")
                self.reportManager.updateReport(self.report)
                exit(0)

            elif option.lower() == "c":
                self.fuzzer.resume()
                return

            elif option.lower() == "n" and not self.directories.empty():
                self.fuzzer.stop()
                return

            elif option.lower() == "s" and len(self.urlList) > 1:
                self.output.newLine()
                raise SkipTargetInterrupt

            else:
                continue

    def processPaths(self):
        while True:
            try:
                while not self.fuzzer.wait(0.25):
                    # Check if the "skip status code" was returned
                    if self.status_skip:
                        self.fuzzer.pause()
                        while self.fuzzer.stopped != len(self.fuzzer.threads):
                            pass

                        self.output.error(
                            "\nSkipped the target due to {0} status code".
                            format(self.status_skip))

                        raise SkipTargetInterrupt

                    elif self.maxtime and time.time(
                    ) - self.startTime > self.maxtime:
                        self.output.error(
                            "\nCanceled because the runtime exceeded the maximal set by user"
                        )
                        exit(0)

                break

            except KeyboardInterrupt:
                self.handlePause(
                    "CTRL+C detected: Pausing threads, please wait...")

    def prepare(self):
        while not self.directories.empty():
            gc.collect()
            self.currentJob += 1
            self.index = 0
            self.currentDirectory = self.directories.get()
            self.output.warning("[{1}] Starting: {0}".format(
                self.currentDirectory, time.strftime("%H:%M:%S")))
            self.fuzzer.requester.basePath = self.output.basePath = self.basePath + self.currentDirectory
            self.fuzzer.start()
            self.processPaths()

        self.report.completed = True
        self.reportManager.updateReport(self.report)
        self.report = None

        return

    def addPort(self, url):
        parsed = urllib.parse.urlparse(url)
        if ":" not in parsed.netloc:
            port = "443" if parsed.scheme == "https" else "80"
            url = url.replace(parsed.netloc, parsed.netloc + ":" + port)

        return url

    def addDirectory(self, path, fullPath=None):
        added = False
        path = path.split("?")[0].split("#")[0]

        if path.rstrip("/") + "/" in [
                directory for directory in self.excludeSubdirs
        ]:
            return False

        fullPath = self.currentDirectory + path if not fullPath else fullPath

        dirs = []

        if self.deep_recursive:
            for i in range(1, path.count("/") + 1):
                dir = fullPath.replace(path, "") + "/".join(
                    path.split("/")[:i])
                dirs.append(dir.rstrip("/") + "/")
        if self.force_recursive:
            if not fullPath.endswith("/"):
                fullPath += "/"
            dirs.append(fullPath)
        elif self.recursive and fullPath.endswith("/"):
            dirs.append(fullPath)

        for dir in dirs:
            if self.scanSubdirs and dir in self.scanSubdirs:
                continue
            elif dir in self.doneDirs:
                continue
            elif self.recursion_depth and dir.count(
                    "/") > self.recursion_depth:
                continue

            self.directories.put(dir)
            self.doneDirs.append(dir)

            self.allJobs += 1
            added = True

        return added

    def addRedirectDirectory(self, path):
        # Resolve the redirect header relative to the current URL and add the
        # path to self.directories if it is a subdirectory of the current URL

        baseUrl = self.currentUrl + self.currentDirectory
        baseUrl = self.addPort(baseUrl)

        absoluteUrl = urllib.parse.urljoin(baseUrl, path.response.redirect)
        absoluteUrl = self.addPort(absoluteUrl)

        if absoluteUrl.startswith(baseUrl) and absoluteUrl != baseUrl:
            path = absoluteUrl[len(baseUrl):]
            fullPath = absoluteUrl[len(self.addPort(self.currentUrl)):]

            return self.addDirectory(path, fullPath)

        return False
Пример #13
0
class Controller(object):
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = (
            open(FileUtils.build_path(script_path, "lib", "controller", "banner.txt"))
            .read()
            .format(**VERSION)
        )

        self.directories = Queue()
        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.savePath = self.script_path
        self.doneDirs = []

        self.urlList = list(filter(None, dict.fromkeys(self.arguments.urlList)))

        self.recursive_level_max = self.arguments.recursive_level_max

        if self.arguments.httpmethod.lower() not in [
            "get", "head", "post", "put", "patch", "options", "delete", "trace", "debug", "connect"
        ]:
            self.output.error("Invalid HTTP method")
            exit(1)

        self.httpmethod = self.arguments.httpmethod.lower()

        if self.arguments.saveHome:
            savePath = self.getSavePath()

            if not FileUtils.exists(savePath):
                FileUtils.create_directory(savePath)

            if FileUtils.exists(savePath) and not FileUtils.is_dir(savePath):
                self.output.error(
                    "Cannot use {} because is a file. Should be a directory".format(
                        savePath
                    )
                )
                exit(1)

            if not FileUtils.can_write(savePath):
                self.output.error("Directory {} is not writable".format(savePath))
                exit(1)

            logs = FileUtils.build_path(savePath, "logs")

            if not FileUtils.exists(logs):
                FileUtils.create_directory(logs)

            reports = FileUtils.build_path(savePath, "reports")

            if not FileUtils.exists(reports):
                FileUtils.create_directory(reports)

            self.savePath = savePath

        self.reportsPath = FileUtils.build_path(self.savePath, "logs")
        self.blacklists = self.getBlacklists()
        self.includeStatusCodes = self.arguments.includeStatusCodes
        self.excludeStatusCodes = self.arguments.excludeStatusCodes
        self.excludeSizes = self.arguments.excludeSizes
        self.excludeTexts = self.arguments.excludeTexts
        self.excludeRegexps = self.arguments.excludeRegexps
        self.recursive = self.arguments.recursive
        self.minimumResponseSize = self.arguments.minimumResponseSize
        self.maximumResponseSize = self.arguments.maximumResponseSize
        self.scanSubdirs = arguments.scanSubdirs
        self.excludeSubdirs = (
            arguments.excludeSubdirs if arguments.excludeSubdirs else []
        )

        self.dictionary = Dictionary(
            self.arguments.wordlist,
            self.arguments.extensions,
            self.arguments.suffixes,
            self.arguments.prefixes,
            self.arguments.lowercase,
            self.arguments.uppercase,
            self.arguments.capitalization,
            self.arguments.forceExtensions,
            self.arguments.excludeExtensions,
            self.arguments.noExtension,
            self.arguments.onlySelected
        )

        self.allJobs = len(self.urlList) * (len(self.scanSubdirs) if self.scanSubdirs else 1)
        self.currentJob = 0
        self.errorLog = None
        self.errorLogPath = None
        self.threadsLock = Lock()
        self.batch = False
        self.batchSession = None

        self.output.header(program_banner)
        self.printConfig()
        self.setupErrorLogs()
        self.output.errorLogFile(self.errorLogPath)

        if self.arguments.autoSave and len(self.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(self.batchDirectoryPath))

        if self.arguments.useRandomAgents:
            self.randomAgents = FileUtils.get_lines(
                FileUtils.build_path(script_path, "db", "user-agents.txt")
            )

        try:
            for url in self.urlList:
                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url
                    self.output.setTarget(self.currentUrl)

                    try:
                        self.requester = Requester(
                            url,
                            cookie=self.arguments.cookie,
                            useragent=self.arguments.useragent,
                            maxPool=self.arguments.threadsCount,
                            maxRetries=self.arguments.maxRetries,
                            timeout=self.arguments.timeout,
                            ip=self.arguments.ip,
                            proxy=self.arguments.proxy,
                            proxylist=self.arguments.proxylist,
                            redirect=self.arguments.redirect,
                            requestByHostname=self.arguments.requestByHostname,
                            httpmethod=self.httpmethod,
                            data=self.arguments.data,
                        )

                        self.requester.request("")

                    except RequestException as e:
                        self.output.error(e.args[0]["message"])
                        raise SkipTargetInterrupt

                    if self.arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    for key, value in arguments.headers.items():
                        self.requester.setHeader(key, value)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath

                    if self.scanSubdirs:
                        for subdir in self.scanSubdirs:
                            self.directories.put(subdir)

                    else:
                        self.directories.put("")

                    self.setupReports(self.requester)

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        testFailPath=self.arguments.testFailPath,
                        threads=self.arguments.threadsCount,
                        delay=self.arguments.delay,
                        matchCallbacks=matchCallbacks,
                        notFoundCallbacks=notFoundCallbacks,
                        errorCallbacks=errorCallbacks,
                    )
                    try:
                        self.wait()
                    except RequestException as e:
                        self.output.error(
                            "Fatal error during site scanning: " + e.args[0]["message"]
                        )
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    continue

        except KeyboardInterrupt:
            self.output.error("\nCanceled by the user")
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

            self.reportManager.close()

        self.output.warning("\nTask Completed")

    def printConfig(self):

        self.output.config(
            ', '.join(self.arguments.extensions),
            ', '.join(self.arguments.prefixes),
            ', '.join(self.arguments.suffixes),
            str(self.arguments.threadsCount),
            str(len(self.dictionary)),
            str(self.httpmethod),
        )

    def getSavePath(self):
        basePath = None
        dirPath = None
        basePath = os.path.expanduser("~")

        if os.name == "nt":
            dirPath = "dirsearch"
        else:
            dirPath = ".dirsearch"

        return FileUtils.build_path(basePath, dirPath)

    def getBlacklists(self):
        reext = re.compile(r'\%ext\%', re.IGNORECASE)
        blacklists = {}

        for status in [400, 403, 500]:
            blacklistFileName = FileUtils.build_path(self.script_path, "db")
            blacklistFileName = FileUtils.build_path(
                blacklistFileName, "{}_blacklist.txt".format(status)
            )

            if not FileUtils.can_read(blacklistFileName):
                # Skip if cannot read file
                continue

            blacklists[status] = []

            for line in FileUtils.get_lines(blacklistFileName):
                # Skip comments
                if line.lstrip().startswith("#"):
                    continue

                if line.startswith("/"):
                    line = line[1:]

                # Classic dirsearch blacklist processing (with %EXT% keyword)
                if "%ext%" in line.lower():
                    for extension in self.arguments.extensions:
                        entry = reext.sub(extension, line)

                        blacklists[status].append(entry)

                # Forced extensions is not used here because -r is only used for wordlist,
                # applying in blacklist may create false negatives

                else:
                    blacklists[status].append(line)

        return blacklists

    def setupErrorLogs(self):
        fileName = "errors-{0}.log".format(time.strftime("%y-%m-%d_%H-%M-%S"))
        self.errorLogPath = FileUtils.build_path(
            FileUtils.build_path(self.savePath, "logs", fileName)
        )
        self.errorLog = open(self.errorLogPath, "w")

    def setupBatchReports(self):
        self.batch = True
        self.batchSession = "BATCH-{0}".format(time.strftime("%y-%m-%d_%H-%M-%S"))
        self.batchDirectoryPath = FileUtils.build_path(
            self.savePath, "reports", self.batchSession
        )

        if not FileUtils.exists(self.batchDirectoryPath):
            FileUtils.create_directory(self.batchDirectoryPath)

            if not FileUtils.exists(self.batchDirectoryPath):
                self.output.error(
                    "Couldn't create batch folder {}".format(self.batchDirectoryPath)
                )
                sys.exit(1)

        if FileUtils.can_write(self.batchDirectoryPath):
            FileUtils.create_directory(self.batchDirectoryPath)
            targetsFile = FileUtils.build_path(self.batchDirectoryPath, "TARGETS.txt")
            FileUtils.write_lines(targetsFile, self.urlList)

        else:
            self.output.error(
                "Couldn't create batch folder {}.".format(self.batchDirectoryPath)
            )
            sys.exit(1)

    def setupReports(self, requester):
        if self.arguments.autoSave:

            basePath = requester.basePath
            basePath = basePath.replace(os.path.sep, ".")[:-1]
            fileName = None
            directoryPath = None

            if self.batch:
                fileName = requester.host
                directoryPath = self.batchDirectoryPath

            else:

                fileName = ('{}_'.format(basePath))
                fileName += time.strftime('%y-%m-%d_%H-%M-%S')
                fileName += ".{0}".format(self.arguments.autoSaveFormat)
                directoryPath = FileUtils.build_path(self.savePath, 'reports', requester.host)

            outputFile = FileUtils.build_path(directoryPath, fileName)

            self.output.outputFile(outputFile)

            if FileUtils.exists(outputFile):
                i = 2

                while FileUtils.exists(outputFile + "_" + str(i)):
                    i += 1

                outputFile += "_" + str(i)

            if not FileUtils.exists(directoryPath):
                FileUtils.create_directory(directoryPath)

                if not FileUtils.exists(directoryPath):
                    self.output.error(
                        "Couldn't create reports folder {}".format(directoryPath)
                    )
                    sys.exit(1)
            if FileUtils.can_write(directoryPath):
                report = None

                if self.arguments.autoSaveFormat == "simple":
                    report = SimpleReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch,
                    )
                elif self.arguments.autoSaveFormat == "json":
                    report = JSONReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch,
                    )
                elif self.arguments.autoSaveFormat == "xml":
                    report = XMLReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch,
                    )
                elif self.arguments.autoSaveFormat == "md":
                    report = MarkdownReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch,
                    )
                elif self.arguments.autoSaveFormat == "csv":
                    report = CSVReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch,
                    )
                else:
                    report = PlainTextReport(
                        requester.host,
                        requester.port,
                        requester.protocol,
                        requester.basePath,
                        outputFile,
                        self.batch
                    )

                self.reportManager.addOutput(report)

            else:
                self.output.error("Can't write reports to {}".format(directoryPath))
                sys.exit(1)

        # TODO: format, refactor code
        if self.arguments.simpleOutputFile:
            self.reportManager.addOutput(
                SimpleReport(
                    requester.host, requester.port, requester.protocol,
                    requester.basePath, self.arguments.simpleOutputFile, self.batch
                )
            )

        if self.arguments.plainTextOutputFile:
            self.reportManager.addOutput(
                PlainTextReport(
                    requester.host, requester.port, requester.protocol,
                    requester.basePath, self.arguments.plainTextOutputFile, self.batch
                )
            )

        if self.arguments.jsonOutputFile:
            self.reportManager.addOutput(
                JSONReport(
                    requester.host, requester.port, requester.protocol,
                    requester.basePath, self.arguments.jsonOutputFile, self.batch
                )
            )

        if self.arguments.xmlOutputFile:
            self.reportManager.addOutput(
                XMLReport(
                    requester.host, requester.port, requester.protocol,
                    requester.basePath, self.arguments.xmlOutputFile, self.batch
                )
            )

        if self.arguments.markdownOutputFile:
            self.reportManager.addOutput(
                MarkdownReport(
                    requester.host, requester.port, requester.protocol,
                    requester.basePath, self.arguments.markdownOutputFile, self.batch
                )
            )
        if self.arguments.csvOutputFile:
            self.reportManager.addOutput(
                CSVReport(
                    requester.host, requester.port, requester.protocol,
                    requester.basePath, self.arguments.csvOutputFile, self.batch
                )
            )

    # TODO: Refactor, this function should be a decorator for all the filters
    def matchCallback(self, path):
        self.index += 1

        if path.status:

            if path.status not in self.excludeStatusCodes and (
                    not self.includeStatusCodes or path.status in self.includeStatusCodes
            ) and (
                    not self.blacklists.get(path.status) or path.path not in self.blacklists.get(path.status)
            ) and (
                    not self.excludeSizes or FileUtils.size_human(len(path.response.body)).strip() not in self.excludeSizes
            ) and (
                    not self.minimumResponseSize or self.minimumResponseSize < len(path.response.body)
            ) and (
                    not self.maximumResponseSize or self.maximumResponseSize > len(path.response.body)
            ):

                for excludeText in self.excludeTexts:
                    if excludeText in path.response.body.decode('iso8859-1'):
                        del path
                        return

                for excludeRegexp in self.excludeRegexps:
                    if (
                        re.search(excludeRegexp, path.response.body.decode('iso8859-1'))
                        is not None
                    ):
                        del path
                        return

                pathIsInScanSubdirs = False
                addedToQueue = False

                if self.scanSubdirs:
                    for subdir in self.scanSubdirs:
                        if subdir == path.path + "/":
                            pathIsInScanSubdirs = True

                if not self.recursive and not pathIsInScanSubdirs and "?" not in path.path:
                    elif path.response.redirect:
                        addedToQueue = self.addRedirectDirectory(path)

                    else:
                        addedToQueue = self.addDirectory(path.path)

                self.output.statusReport(
                    path.path, path.response, self.arguments.full_url, addedToQueue
                )

                if self.arguments.matches_proxy:
                    self.requester.request(path.path, proxy=self.arguments.matches_proxy)

                newPath = "{}{}".format(self.currentDirectory, path.path)

                self.reportManager.addPath(newPath, path.status, path.response)

                self.reportManager.save()

                del path
Пример #14
0
    def __init__(self, script_path, arguments, output):
        global VERSION
        PROGRAM_BANNER = open(FileUtils.buildPath(script_path, "lib", "controller", "banner.txt")).read().format(
            **VERSION)
        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.blacklists = self.getBlacklists()
        self.fuzzer = None
        self.excludeStatusCodes = self.arguments.excludeStatusCodes
        self.recursive = self.arguments.recursive
        self.directories = Queue()
        self.excludeSubdirs = (arguments.excludeSubdirs if arguments.excludeSubdirs is not None else [])
        self.output.header(PROGRAM_BANNER)
        self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions,
                                     self.arguments.lowercase)
        self.printConfig()
        self.errorLog = None
        self.errorLogPath = None
        self.errorLogLock = Lock()
        self.batch = False
        self.batchSession = None
        self.setupErrorLogs()
        self.output.newLine("\nError Log: {0}".format(self.errorLogPath))
        if self.arguments.autoSave and len(self.arguments.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(self.batchDirectoryPath))
        if self.arguments.useRandomAgents:
            self.randomAgents = FileUtils.getLines(FileUtils.buildPath(script_path, "db", "user-agents.txt"))
        try:
            for url in self.arguments.urlList:
                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url

                    self.requester = Requester(url, cookie=self.arguments.cookie,
                                               useragent=self.arguments.useragent, maxPool=self.arguments.threadsCount,
                                               maxRetries=self.arguments.maxRetries, timeout=self.arguments.timeout,
                                               ip=self.arguments.ip, proxy=self.arguments.proxy,
                                               redirect=self.arguments.redirect)
                    if self.arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)
                    for key, value in arguments.headers.items():
                        self.requester.setHeader(key, value)
                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath
                    if self.arguments.scanSubdirs is not None:
                        for subdir in self.arguments.scanSubdirs:
                            self.directories.put(subdir)
                    else:
                        self.directories.put('')
                    self.setupReports(self.requester)

                    self.output.target(self.currentUrl)
                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]
                    self.fuzzer = Fuzzer(self.requester, self.dictionary, testFailPath=self.arguments.testFailPath,
                                         threads=self.arguments.threadsCount, matchCallbacks=matchCallbacks,
                                         notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks)
                    self.wait()
                except SkipTargetInterrupt:
                    continue
                finally:
                    self.reportManager.save()
        except KeyboardInterrupt:
            self.output.error('\nCanceled by the user')
            exit(0)
        finally:
            if not self.errorLog.closed:
                self.errorLog.close()
            self.reportManager.close()

        self.output.warning('\nTask Completed')
Пример #15
0
class Controller(object):
    def __init__(self, script_path, arguments, output):
        global VERSION
        PROGRAM_BANNER = open(FileUtils.buildPath(script_path, "lib", "controller", "banner.txt")).read().format(
            **VERSION)
        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.blacklists = self.getBlacklists()
        self.fuzzer = None
        self.excludeStatusCodes = self.arguments.excludeStatusCodes
        self.recursive = self.arguments.recursive
        self.directories = Queue()
        self.excludeSubdirs = (arguments.excludeSubdirs if arguments.excludeSubdirs is not None else [])
        self.output.header(PROGRAM_BANNER)
        self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions,
                                     self.arguments.lowercase)
        self.printConfig()
        self.errorLog = None
        self.errorLogPath = None
        self.errorLogLock = Lock()
        self.batch = False
        self.batchSession = None
        self.setupErrorLogs()
        self.output.newLine("\nError Log: {0}".format(self.errorLogPath))
        if self.arguments.autoSave and len(self.arguments.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(self.batchDirectoryPath))
        if self.arguments.useRandomAgents:
            self.randomAgents = FileUtils.getLines(FileUtils.buildPath(script_path, "db", "user-agents.txt"))
        try:
            for url in self.arguments.urlList:
                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url

                    self.requester = Requester(url, cookie=self.arguments.cookie,
                                               useragent=self.arguments.useragent, maxPool=self.arguments.threadsCount,
                                               maxRetries=self.arguments.maxRetries, timeout=self.arguments.timeout,
                                               ip=self.arguments.ip, proxy=self.arguments.proxy,
                                               redirect=self.arguments.redirect)
                    if self.arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)
                    for key, value in arguments.headers.items():
                        self.requester.setHeader(key, value)
                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath
                    if self.arguments.scanSubdirs is not None:
                        for subdir in self.arguments.scanSubdirs:
                            self.directories.put(subdir)
                    else:
                        self.directories.put('')
                    self.setupReports(self.requester)

                    self.output.target(self.currentUrl)
                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]
                    self.fuzzer = Fuzzer(self.requester, self.dictionary, testFailPath=self.arguments.testFailPath,
                                         threads=self.arguments.threadsCount, matchCallbacks=matchCallbacks,
                                         notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks)
                    self.wait()
                except SkipTargetInterrupt:
                    continue
                finally:
                    self.reportManager.save()
        except KeyboardInterrupt:
            self.output.error('\nCanceled by the user')
            exit(0)
        finally:
            if not self.errorLog.closed:
                self.errorLog.close()
            self.reportManager.close()

        self.output.warning('\nTask Completed')

    def printConfig(self):
        self.output.config(', '.join(self.arguments.extensions), str(self.arguments.threadsCount),
                                str(len(self.dictionary)))

    def getBlacklists(self):
        blacklists = {}
        for status in [400, 403, 500]:
            blacklistFileName = FileUtils.buildPath(self.script_path, 'db')
            blacklistFileName = FileUtils.buildPath(blacklistFileName, '{}_blacklist.txt'.format(status))
            if not FileUtils.canRead(blacklistFileName):
                # Skip if cannot read file
                continue
            blacklists[status] = []
            for line in FileUtils.getLines(blacklistFileName):
                # Skip comments
                if line.lstrip().startswith('#'):
                    continue
                blacklists[status].append(line)
        return blacklists

    def setupErrorLogs(self):
        fileName = "errors-{0}.log".format(time.strftime('%y-%m-%d_%H-%M-%S'))
        self.errorLogPath = FileUtils.buildPath(self.script_path, 'logs', fileName)
        self.errorLog = open(self.errorLogPath, "w")

    def setupBatchReports(self):
        self.batch = True
        self.batchSession = "BATCH-{0}".format(time.strftime('%y-%m-%d_%H-%M-%S'))
        self.batchDirectoryPath = FileUtils.buildPath(self.script_path, 'reports', self.batchSession)
        if not FileUtils.exists(self.batchDirectoryPath):
            FileUtils.createDirectory(self.batchDirectoryPath)
            if not FileUtils.exists(self.batchDirectoryPath):
                self.output.error("Couldn't create batch folder {}".format(self.batchDirectoryPath))
                sys.exit(1)
        if FileUtils.canWrite(self.batchDirectoryPath):
            FileUtils.createDirectory(self.batchDirectoryPath)
            targetsFile = FileUtils.buildPath(self.batchDirectoryPath, "TARGETS.txt")
            FileUtils.writeLines(targetsFile, self.arguments.urlList)
        else:
            self.output.error("Couldn't create batch folder {}.".format(self.batchDirectoryPath))
            sys.exit(1)

    def setupReports(self, requester):
        if self.arguments.autoSave:
            basePath = ('/' if requester.basePath is '' else requester.basePath)
            basePath = basePath.replace(os.path.sep, '.')[1:-1]
            fileName = None
            directoryPath = None
            if self.batch:
                fileName = requester.host
                directoryPath = self.batchDirectoryPath
            else:
                fileName = ('{}_'.format(basePath) if basePath is not '' else '')
                fileName += time.strftime('%y-%m-%d_%H-%M-%S')
                directoryPath = FileUtils.buildPath(self.script_path, 'reports', requester.host)
            outputFile = FileUtils.buildPath(directoryPath, fileName)

            if FileUtils.exists(outputFile):
                i = 2
                while FileUtils.exists(outputFile + "_" + str(i)):
                    i += 1
                outputFile += "_" + str(i)
            if not FileUtils.exists(directoryPath):
                FileUtils.createDirectory(directoryPath)
                if not FileUtils.exists(directoryPath):
                    self.output.error("Couldn't create reports folder {}".format(directoryPath))
                    sys.exit(1)
            if FileUtils.canWrite(directoryPath):
                report = None
                if self.arguments.autoSaveFormat == 'simple':
                    report = SimpleReport(requester.host, requester.port, requester.protocol, requester.basePath,
                                          outputFile)
                if self.arguments.autoSaveFormat == 'json':
                    report = JSONReport(requester.host, requester.port, requester.protocol, requester.basePath,
                                        outputFile)
                else:
                    report = PlainTextReport(requester.host, requester.port, requester.protocol, requester.basePath,
                                             outputFile)
                self.reportManager.addOutput(report)
            else:
                self.output.error("Can't write reports to {}".format(directoryPath))
                sys.exit(1)
        if self.arguments.simpleOutputFile is not None:
            self.reportManager.addOutput(SimpleReport(requester.host, requester.port, requester.protocol,
                                                      requester.basePath, self.arguments.simpleOutputFile))
        if self.arguments.plainTextOutputFile is not None:
            self.reportManager.addOutput(PlainTextReport(requester.host, requester.port, requester.protocol,
                                                         requester.basePath, self.arguments.plainTextOutputFile))
        if self.arguments.jsonOutputFile is not None:
            self.reportManager.addOutput(JSONReport(requester.host, requester.port, requester.protocol,
                                                    requester.basePath, self.arguments.jsonOutputFile))

    def matchCallback(self, path):
        self.index += 1
        if path.status is not None:
            if path.status not in self.excludeStatusCodes and (
                            self.blacklists.get(path.status) is None or path.path not in self.blacklists.get(
                        path.status)):
                self.output.statusReport(path.path, path.response)
                self.addDirectory(path.path)
                self.reportManager.addPath(self.currentDirectory + path.path, path.status, path.response)
                self.reportManager.save()
                del path

    def notFoundCallback(self, path):
        self.index += 1
        self.output.lastPath(path, self.index, len(self.dictionary))
        del path

    def errorCallback(self, path, errorMsg):
        self.output.addConnectionError()
        del path

    def appendErrorLog(self, path, errorMsg):
        with self.errorLogLock:
            line = time.strftime('[%y-%m-%d %H:%M:%S] - ')
            line += self.currentUrl + " - " + path + " - " + errorMsg
            self.errorLog.write(os.linesep + line)
            self.errorLog.flush()

    def handleInterrupt(self):
        self.output.warning('CTRL+C detected: Pausing threads, please wait...')
        self.fuzzer.pause()
        try:
            while True:
                msg = "[e]xit / [c]ontinue"
                if not self.directories.empty():
                    msg += " / [n]ext"
                if len(self.arguments.urlList) > 1:
                    msg += " / [s]kip target"
                self.output.inLine(msg + ': ')

                option = input()
                if option.lower() == 'e':
                    self.exit = True
                    self.fuzzer.stop()
                    raise KeyboardInterrupt
                elif option.lower() == 'c':
                    self.fuzzer.play()
                    return
                elif not self.directories.empty() and option.lower() == 'n':
                    self.fuzzer.stop()
                    return
                elif len(self.arguments.urlList) > 1 and option.lower() == 's':
                    raise SkipTargetInterrupt
                else:
                    continue
        except KeyboardInterrupt as SystemExit:
            self.exit = True
            raise KeyboardInterrupt


    def processPaths(self):
        while True:
            try:
                while not self.fuzzer.wait(0.3):
                    continue
                break
            except (KeyboardInterrupt, SystemExit) as e:
                self.handleInterrupt()

    def wait(self):
        while not self.directories.empty():
            self.index = 0
            self.currentDirectory = self.directories.get()
            self.output.warning('[{1}] Starting: {0}'.format(self.currentDirectory, time.strftime('%H:%M:%S')))
            self.fuzzer.requester.basePath = self.basePath + self.currentDirectory
            self.output.basePath = self.basePath + self.currentDirectory
            self.fuzzer.start()
            self.processPaths()
        return

    def addDirectory(self, path):
        if not self.recursive:
            return False
        if path.endswith('/'):
            if path in [directory + '/' for directory in self.excludeSubdirs]:
                return False
            self.directories.put(self.currentDirectory + path)
            return True
        else:
            return False
Пример #16
0
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = open(FileUtils.buildPath(script_path, "lib", "controller", "banner.txt")).read().format(
            **VERSION)

        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.savePath = self.script_path
        self.doneDirs = []

        self.recursive_level_max = self.arguments.recursive_level_max

        if self.arguments.httpmethod.lower() not in ["get", "head", "post"]:
            self.output.error("Inavlid http method!")
            exit(1)

        self.httpmethod = self.arguments.httpmethod.lower()

        if self.arguments.saveHome:
            savePath = self.getSavePath()

            if not FileUtils.exists(savePath):
                FileUtils.createDirectory(savePath)

            if FileUtils.exists(savePath) and not FileUtils.isDir(savePath):
                self.output.error('Cannot use {} because is a file. Should be a directory'.format(savePath))
                exit(1)

            if not FileUtils.canWrite(savePath):
                self.output.error('Directory {} is not writable'.format(savePath))
                exit(1)

            logs = FileUtils.buildPath(savePath, "logs")

            if not FileUtils.exists(logs):
                FileUtils.createDirectory(logs)

            reports = FileUtils.buildPath(savePath, "reports")

            if not FileUtils.exists(reports):
                FileUtils.createDirectory(reports)

            self.savePath = savePath

        self.reportsPath = FileUtils.buildPath(self.savePath, "logs")
        self.blacklists = self.getBlacklists()
        self.fuzzer = None
        self.excludeStatusCodes = self.arguments.excludeStatusCodes
        self.recursive = self.arguments.recursive
        self.suppressEmpty = self.arguments.suppressEmpty
        self.directories = Queue()
        self.excludeSubdirs = (arguments.excludeSubdirs if arguments.excludeSubdirs is not None else [])
        self.output.header(program_banner)
        self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions,
                                     self.arguments.lowercase, self.arguments.forceExtensions)
        self.printConfig()
        self.errorLog = None
        self.errorLogPath = None
        self.errorLogLock = Lock()
        self.batch = False
        self.batchSession = None
        self.setupErrorLogs()
        self.output.newLine("\nError Log: {0}".format(self.errorLogPath))

        if self.arguments.autoSave and len(self.arguments.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(self.batchDirectoryPath))

        if self.arguments.useRandomAgents:
            self.randomAgents = FileUtils.getLines(FileUtils.buildPath(script_path, "db", "user-agents.txt"))

        try:
            for url in self.arguments.urlList:

                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url
                    self.output.target(self.currentUrl)

                    try:
                        self.requester = Requester(url, cookie=self.arguments.cookie,
                                                   useragent=self.arguments.useragent,
                                                   maxPool=self.arguments.threadsCount,
                                                   maxRetries=self.arguments.maxRetries, delay=self.arguments.delay,
                                                   timeout=self.arguments.timeout,
                                                   ip=self.arguments.ip, proxy=self.arguments.proxy,
                                                   redirect=self.arguments.redirect,
                                                   requestByHostname=self.arguments.requestByHostname,
                                                   httpmethod=self.httpmethod)
                        self.requester.request("/")

                    except RequestException as e:
                        self.output.error(e.args[0]['message'])
                        raise SkipTargetInterrupt

                    if self.arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    for key, value in arguments.headers.items():
                        self.requester.setHeader(key, value)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath

                    if self.arguments.scanSubdirs is not None:
                        for subdir in self.arguments.scanSubdirs:
                            self.directories.put(subdir)

                    else:
                        self.directories.put('')

                    self.setupReports(self.requester)

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(self.requester, self.dictionary, testFailPath=self.arguments.testFailPath,
                                         threads=self.arguments.threadsCount, matchCallbacks=matchCallbacks,
                                         notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks)
                    try:
                        self.wait()
                    except RequestException as e:
                        self.output.error("Fatal error during site scanning: " + e.args[0]['message'])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    continue

                finally:
                    self.reportManager.save()

        except KeyboardInterrupt:
            self.output.error('\nCanceled by the user')
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

            self.reportManager.close()

        self.output.warning('\nTask Completed')