def getBytes(value, encoding=UNICODE_ENCODING, errors="strict", unsafe=True): """ Returns byte representation of provided Unicode value >>> getBytes(u"foo\\\\x01\\\\x83\\\\xffbar") == b"foo\\x01\\x83\\xffbar" True """ retVal = value if isinstance(value, six.text_type): if INVALID_UNICODE_PRIVATE_AREA: if unsafe: for char in xrange(0xF0000, 0xF00FF + 1): value = value.replace( _unichr(char), "%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000)) retVal = value.encode(encoding, errors) if unsafe: retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: decodeHex(_.group(1)), retVal) else: retVal = value.encode(encoding, errors) if unsafe: retVal = re.sub(b"\\\\x([0-9a-f]{2})", lambda _: decodeHex(_.group(1)), retVal) return retVal
def _(match): retVal = match.group(0) try: retVal = _unichr(int(match.group(1))) except (ValueError, OverflowError): pass return retVal
def _reversible(ex): if INVALID_UNICODE_PRIVATE_AREA: return (u"".join( _unichr( int('000f00%2x' % (_ if isinstance(_, int) else ord(_)), 16)) for _ in ex.object[ex.start:ex.end]), ex.end) else: return (u"".join(INVALID_UNICODE_CHAR_FORMAT % (_ if isinstance(_, int) else ord(_)) for _ in ex.object[ex.start:ex.end]), ex.end)
def htmlUnescape(value): """ Returns (basic conversion) HTML unescaped value >>> htmlUnescape('a<b') == 'a<b' True """ retVal = value if value and isinstance(value, six.string_types): replacements = (("<", '<'), (">", '>'), (""", '"'), (" ", ' '), ("&", '&'), ("'", "'")) for code, value in replacements: retVal = retVal.replace(code, value) try: retVal = re.sub(r"&#x([^ ;]+);", lambda match: _unichr(int(match.group(1), 16)), retVal) except (ValueError, OverflowError): pass return retVal
def purge(directory): """ Safely removes content from a given directory """ if not os.path.isdir(directory): warnMsg = "skipping purging of directory '%s' as it does not exist" % directory logger.warn(warnMsg) return infoMsg = "purging content of directory '%s'..." % directory logger.info(infoMsg) filepaths = [] dirpaths = [] for rootpath, directories, filenames in os.walk(directory): dirpaths.extend( os.path.abspath(os.path.join(rootpath, _)) for _ in directories) filepaths.extend( os.path.abspath(os.path.join(rootpath, _)) for _ in filenames) logger.debug("changing file attributes") for filepath in filepaths: try: os.chmod(filepath, stat.S_IREAD | stat.S_IWRITE) except: pass logger.debug("writing random data to files") for filepath in filepaths: try: filesize = os.path.getsize(filepath) with openFile(filepath, "w+b") as f: f.write("".join( _unichr(random.randint(0, 255)) for _ in xrange(filesize))) except: pass logger.debug("truncating files") for filepath in filepaths: try: with open(filepath, 'w') as f: pass except: pass logger.debug("renaming filenames to random values") for filepath in filepaths: try: os.rename( filepath, os.path.join( os.path.dirname(filepath), "".join( random.sample(string.ascii_letters, random.randint(4, 8))))) except: pass dirpaths.sort(key=functools.cmp_to_key( lambda x, y: y.count(os.path.sep) - x.count(os.path.sep))) logger.debug("renaming directory names to random values") for dirpath in dirpaths: try: os.rename( dirpath, os.path.join( os.path.dirname(dirpath), "".join( random.sample(string.ascii_letters, random.randint(4, 8))))) except: pass logger.debug("deleting the whole directory tree") os.chdir(os.path.join(directory, "..")) try: shutil.rmtree(directory) except OSError as ex: logger.error("problem occurred while removing directory '%s' ('%s')" % (directory, getSafeExString(ex)))
def pivotDumpTable(table, colList, count=None, blind=True, alias=None): lengths = {} entries = {} dumpNode = queries[Backend.getIdentifiedDbms()].dump_table.blind validColumnList = False validPivotValue = False if count is None: query = dumpNode.count % table query = agent.whereQuery(query) count = inject.getValue( query, union=False, error=False, expected=EXPECTED.INT, charsetType=CHARSET_TYPE.DIGITS) if blind else inject.getValue( query, blind=False, time=False, expected=EXPECTED.INT) if hasattr(count, "isdigit") and count.isdigit(): count = int(count) if count == 0: infoMsg = "table '%s' appears to be empty" % unsafeSQLIdentificatorNaming( table) logger.info(infoMsg) for column in colList: lengths[column] = len(column) entries[column] = [] return entries, lengths elif not isNumPosStrValue(count): return None for column in colList: lengths[column] = 0 entries[column] = BigArray() colList = filterNone( sorted(colList, key=lambda x: len(x) if x else MAX_INT)) if conf.pivotColumn: for _ in colList: if re.search(r"(.+\.)?%s" % re.escape(conf.pivotColumn), _, re.I): infoMsg = "using column '%s' as a pivot " % conf.pivotColumn infoMsg += "for retrieving row data" logger.info(infoMsg) colList.remove(_) colList.insert(0, _) validPivotValue = True break if not validPivotValue: warnMsg = "column '%s' not " % conf.pivotColumn warnMsg += "found in table '%s'" % table logger.warn(warnMsg) if not validPivotValue: for column in colList: infoMsg = "fetching number of distinct " infoMsg += "values for column '%s'" % column.replace( ("%s." % alias) if alias else "", "") logger.info(infoMsg) query = dumpNode.count2 % (column, table) query = agent.whereQuery(query) value = inject.getValue(query, blind=blind, union=not blind, error=not blind, expected=EXPECTED.INT, charsetType=CHARSET_TYPE.DIGITS) if isNumPosStrValue(value): validColumnList = True if value == count: infoMsg = "using column '%s' as a pivot " % column.replace( ("%s." % alias) if alias else "", "") infoMsg += "for retrieving row data" logger.info(infoMsg) validPivotValue = True colList.remove(column) colList.insert(0, column) break if not validColumnList: errMsg = "all provided column name(s) are non-existent" raise SqlmapNoneDataException(errMsg) if not validPivotValue: warnMsg = "no proper pivot column provided (with unique values)." warnMsg += " It won't be possible to retrieve all rows" logger.warn(warnMsg) pivotValue = " " breakRetrieval = False def _(column, pivotValue): if column == colList[0]: query = dumpNode.query.replace( "'%s'" if unescaper.escape(pivotValue, False) != pivotValue else "%s", "%s") % (agent.preprocessField( table, column), table, agent.preprocessField( table, column), unescaper.escape(pivotValue, False)) else: query = dumpNode.query2.replace( "'%s'" if unescaper.escape(pivotValue, False) != pivotValue else "%s", "%s") % (agent.preprocessField(table, column), table, agent.preprocessField(table, colList[0]), unescaper.escape(pivotValue, False)) query = agent.whereQuery(query) return unArrayizeValue( inject.getValue(query, blind=blind, time=blind, union=not blind, error=not blind)) try: for i in xrange(count): if breakRetrieval: break for column in colList: value = _(column, pivotValue) if column == colList[0]: if isNoneValue(value): try: for pivotValue in filterNone( (" " if pivotValue == " " else None, "%s%s" % (pivotValue[0], _unichr(ord(pivotValue[1]) + 1)) if len(pivotValue) > 1 else None, _unichr(ord(pivotValue[0]) + 1))): value = _(column, pivotValue) if not isNoneValue(value): break except ValueError: pass if isNoneValue(value) or value == NULL: breakRetrieval = True break pivotValue = safechardecode(value) if conf.limitStart or conf.limitStop: if conf.limitStart and (i + 1) < conf.limitStart: warnMsg = "skipping first %d pivot " % conf.limitStart warnMsg += "point values" singleTimeWarnMessage(warnMsg) break elif conf.limitStop and (i + 1) > conf.limitStop: breakRetrieval = True break value = "" if isNoneValue(value) else unArrayizeValue(value) lengths[column] = max( lengths[column], len( DUMP_REPLACEMENTS.get(getUnicode(value), getUnicode(value)))) entries[column].append(value) except KeyboardInterrupt: kb.dumpKeyboardInterrupt = True warnMsg = "user aborted during enumeration. sqlmap " warnMsg += "will display partial output" logger.warn(warnMsg) except SqlmapConnectionException as ex: errMsg = "connection exception detected ('%s'). sqlmap " % getSafeExString( ex) errMsg += "will display partial output" logger.critical(errMsg) return entries, lengths
def decodePage(page, contentEncoding, contentType, percentDecode=True): """ Decode compressed/charset HTTP response >>> getText(decodePage(b"<html>foo&bar</html>", None, "text/html; charset=utf-8")) '<html>foo&bar</html>' """ if not page or (conf.nullConnection and len(page) < 2): return getUnicode(page) if hasattr(contentEncoding, "lower"): contentEncoding = contentEncoding.lower() else: contentEncoding = "" if hasattr(contentType, "lower"): contentType = contentType.lower() else: contentType = "" if contentEncoding in ("gzip", "x-gzip", "deflate"): if not kb.pageCompress: return None try: if contentEncoding == "deflate": data = io.BytesIO( zlib.decompress(page, -15) ) # Reference: http://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations else: data = gzip.GzipFile("", "rb", 9, io.BytesIO(page)) size = struct.unpack( "<l", page[-4:] )[0] # Reference: http://pydoc.org/get.cgi/usr/local/lib/python2.5/gzip.py if size > MAX_CONNECTION_TOTAL_SIZE: raise Exception("size too large") page = data.read() except Exception as ex: if "<html" not in page: # in some cases, invalid "Content-Encoding" appears for plain HTML (should be ignored) errMsg = "detected invalid data for declared content " errMsg += "encoding '%s' ('%s')" % (contentEncoding, getSafeExString(ex)) singleTimeLogMessage(errMsg, logging.ERROR) warnMsg = "turning off page compression" singleTimeWarnMessage(warnMsg) kb.pageCompress = False raise SqlmapCompressionException if not conf.encoding: httpCharset, metaCharset = None, None # Reference: http://stackoverflow.com/questions/1020892/python-urllib2-read-to-unicode if contentType.find("charset=") != -1: httpCharset = checkCharEncoding(contentType.split("charset=")[-1]) metaCharset = checkCharEncoding( extractRegexResult(META_CHARSET_REGEX, page)) if (any((httpCharset, metaCharset)) and not all( (httpCharset, metaCharset))) or (httpCharset == metaCharset and all( (httpCharset, metaCharset))): kb.pageEncoding = httpCharset or metaCharset # Reference: http://bytes.com/topic/html-css/answers/154758-http-equiv-vs-true-header-has-precedence debugMsg = "declared web page charset '%s'" % kb.pageEncoding singleTimeLogMessage(debugMsg, logging.DEBUG, debugMsg) else: kb.pageEncoding = None else: kb.pageEncoding = conf.encoding # can't do for all responses because we need to support binary files too if isinstance(page, six.binary_type) and "text/" in contentType: # e.g. 	Ãëàâà if b"&#" in page: page = re.sub( b"&#x([0-9a-f]{1,2});", lambda _: decodeHex( _.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)), page) page = re.sub( b"&#(\\d{1,3});", lambda _: six.int2byte(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page) # e.g. %20%28%29 if percentDecode: if b"%" in page: page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page) # e.g. & page = re.sub( b"&([^;]+);", lambda _: six.int2byte(HTML_ENTITIES[getText(_.group(1))]) if HTML_ENTITIES.get(getText(_.group(1)), 256) < 256 else _.group( 0), page) kb.pageEncoding = kb.pageEncoding or checkCharEncoding( getHeuristicCharEncoding(page)) if (kb.pageEncoding or "").lower() == "utf-8-sig": kb.pageEncoding = "utf-8" if page and page.startswith( "\xef\xbb\xbf" ): # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling) page = page[3:] page = getUnicode(page, kb.pageEncoding) # e.g. ’…™ if "&#" in page: def _(match): retVal = match.group(0) try: retVal = _unichr(int(match.group(1))) except (ValueError, OverflowError): pass return retVal page = re.sub(r"&#(\d+);", _, page) # e.g. ζ page = re.sub( r"&([^;]+);", lambda _: _unichr(HTML_ENTITIES[_.group(1)]) if HTML_ENTITIES.get(_.group(1), 0) > 255 else _.group(0), page) return page