def getBytes(value, encoding=UNICODE_ENCODING, errors="strict", unsafe=True):
"""
Returns byte representation of provided Unicode value
>>> getBytes(u"foo\\\\x01\\\\x83\\\\xffbar") == b"foo\\x01\\x83\\xffbar"
True
"""
retVal = value
if isinstance(value, six.text_type):
if INVALID_UNICODE_PRIVATE_AREA:
if unsafe:
for char in xrange(0xF0000, 0xF00FF + 1):
value = value.replace(
_unichr(char),
"%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000))
retVal = value.encode(encoding, errors)
if unsafe:
retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER,
lambda _: decodeHex(_.group(1)), retVal)
else:
retVal = value.encode(encoding, errors)
if unsafe:
retVal = re.sub(b"\\\\x([0-9a-f]{2})",
lambda _: decodeHex(_.group(1)), retVal)
return retVal
def _(match):
retVal = match.group(0)
try:
retVal = _unichr(int(match.group(1)))
except (ValueError, OverflowError):
pass
return retVal
def _reversible(ex):
if INVALID_UNICODE_PRIVATE_AREA:
return (u"".join(
_unichr(
int('000f00%2x' % (_ if isinstance(_, int) else ord(_)), 16))
for _ in ex.object[ex.start:ex.end]), ex.end)
else:
return (u"".join(INVALID_UNICODE_CHAR_FORMAT %
(_ if isinstance(_, int) else ord(_))
for _ in ex.object[ex.start:ex.end]), ex.end)
def htmlUnescape(value):
"""
Returns (basic conversion) HTML unescaped value
>>> htmlUnescape('a<b') == 'a<b'
True
"""
retVal = value
if value and isinstance(value, six.string_types):
replacements = (("<", '<'), (">", '>'), (""", '"'), (" ", ' '), ("&", '&'), ("'", "'"))
for code, value in replacements:
retVal = retVal.replace(code, value)
try:
retVal = re.sub(r"&#x([^ ;]+);", lambda match: _unichr(int(match.group(1), 16)), retVal)
except (ValueError, OverflowError):
pass
return retVal
def purge(directory):
"""
Safely removes content from a given directory
"""
if not os.path.isdir(directory):
warnMsg = "skipping purging of directory '%s' as it does not exist" % directory
logger.warn(warnMsg)
return
infoMsg = "purging content of directory '%s'..." % directory
logger.info(infoMsg)
filepaths = []
dirpaths = []
for rootpath, directories, filenames in os.walk(directory):
dirpaths.extend(
os.path.abspath(os.path.join(rootpath, _)) for _ in directories)
filepaths.extend(
os.path.abspath(os.path.join(rootpath, _)) for _ in filenames)
logger.debug("changing file attributes")
for filepath in filepaths:
try:
os.chmod(filepath, stat.S_IREAD | stat.S_IWRITE)
except:
pass
logger.debug("writing random data to files")
for filepath in filepaths:
try:
filesize = os.path.getsize(filepath)
with openFile(filepath, "w+b") as f:
f.write("".join(
_unichr(random.randint(0, 255)) for _ in xrange(filesize)))
except:
pass
logger.debug("truncating files")
for filepath in filepaths:
try:
with open(filepath, 'w') as f:
pass
except:
pass
logger.debug("renaming filenames to random values")
for filepath in filepaths:
try:
os.rename(
filepath,
os.path.join(
os.path.dirname(filepath), "".join(
random.sample(string.ascii_letters,
random.randint(4, 8)))))
except:
pass
dirpaths.sort(key=functools.cmp_to_key(
lambda x, y: y.count(os.path.sep) - x.count(os.path.sep)))
logger.debug("renaming directory names to random values")
for dirpath in dirpaths:
try:
os.rename(
dirpath,
os.path.join(
os.path.dirname(dirpath), "".join(
random.sample(string.ascii_letters,
random.randint(4, 8)))))
except:
pass
logger.debug("deleting the whole directory tree")
os.chdir(os.path.join(directory, ".."))
try:
shutil.rmtree(directory)
except OSError as ex:
logger.error("problem occurred while removing directory '%s' ('%s')" %
(directory, getSafeExString(ex)))
def pivotDumpTable(table, colList, count=None, blind=True, alias=None):
lengths = {}
entries = {}
dumpNode = queries[Backend.getIdentifiedDbms()].dump_table.blind
validColumnList = False
validPivotValue = False
if count is None:
query = dumpNode.count % table
query = agent.whereQuery(query)
count = inject.getValue(
query,
union=False,
error=False,
expected=EXPECTED.INT,
charsetType=CHARSET_TYPE.DIGITS) if blind else inject.getValue(
query, blind=False, time=False, expected=EXPECTED.INT)
if hasattr(count, "isdigit") and count.isdigit():
count = int(count)
if count == 0:
infoMsg = "table '%s' appears to be empty" % unsafeSQLIdentificatorNaming(
table)
logger.info(infoMsg)
for column in colList:
lengths[column] = len(column)
entries[column] = []
return entries, lengths
elif not isNumPosStrValue(count):
return None
for column in colList:
lengths[column] = 0
entries[column] = BigArray()
colList = filterNone(
sorted(colList, key=lambda x: len(x) if x else MAX_INT))
if conf.pivotColumn:
for _ in colList:
if re.search(r"(.+\.)?%s" % re.escape(conf.pivotColumn), _, re.I):
infoMsg = "using column '%s' as a pivot " % conf.pivotColumn
infoMsg += "for retrieving row data"
logger.info(infoMsg)
colList.remove(_)
colList.insert(0, _)
validPivotValue = True
break
if not validPivotValue:
warnMsg = "column '%s' not " % conf.pivotColumn
warnMsg += "found in table '%s'" % table
logger.warn(warnMsg)
if not validPivotValue:
for column in colList:
infoMsg = "fetching number of distinct "
infoMsg += "values for column '%s'" % column.replace(
("%s." % alias) if alias else "", "")
logger.info(infoMsg)
query = dumpNode.count2 % (column, table)
query = agent.whereQuery(query)
value = inject.getValue(query,
blind=blind,
union=not blind,
error=not blind,
expected=EXPECTED.INT,
charsetType=CHARSET_TYPE.DIGITS)
if isNumPosStrValue(value):
validColumnList = True
if value == count:
infoMsg = "using column '%s' as a pivot " % column.replace(
("%s." % alias) if alias else "", "")
infoMsg += "for retrieving row data"
logger.info(infoMsg)
validPivotValue = True
colList.remove(column)
colList.insert(0, column)
break
if not validColumnList:
errMsg = "all provided column name(s) are non-existent"
raise SqlmapNoneDataException(errMsg)
if not validPivotValue:
warnMsg = "no proper pivot column provided (with unique values)."
warnMsg += " It won't be possible to retrieve all rows"
logger.warn(warnMsg)
pivotValue = " "
breakRetrieval = False
def _(column, pivotValue):
if column == colList[0]:
query = dumpNode.query.replace(
"'%s'" if unescaper.escape(pivotValue, False) != pivotValue
else "%s", "%s") % (agent.preprocessField(
table, column), table, agent.preprocessField(
table, column), unescaper.escape(pivotValue, False))
else:
query = dumpNode.query2.replace(
"'%s'"
if unescaper.escape(pivotValue, False) != pivotValue else "%s",
"%s") % (agent.preprocessField(table, column), table,
agent.preprocessField(table, colList[0]),
unescaper.escape(pivotValue, False))
query = agent.whereQuery(query)
return unArrayizeValue(
inject.getValue(query,
blind=blind,
time=blind,
union=not blind,
error=not blind))
try:
for i in xrange(count):
if breakRetrieval:
break
for column in colList:
value = _(column, pivotValue)
if column == colList[0]:
if isNoneValue(value):
try:
for pivotValue in filterNone(
(" " if pivotValue == " " else None,
"%s%s" % (pivotValue[0],
_unichr(ord(pivotValue[1]) + 1))
if len(pivotValue) > 1 else None,
_unichr(ord(pivotValue[0]) + 1))):
value = _(column, pivotValue)
if not isNoneValue(value):
break
except ValueError:
pass
if isNoneValue(value) or value == NULL:
breakRetrieval = True
break
pivotValue = safechardecode(value)
if conf.limitStart or conf.limitStop:
if conf.limitStart and (i + 1) < conf.limitStart:
warnMsg = "skipping first %d pivot " % conf.limitStart
warnMsg += "point values"
singleTimeWarnMessage(warnMsg)
break
elif conf.limitStop and (i + 1) > conf.limitStop:
breakRetrieval = True
break
value = "" if isNoneValue(value) else unArrayizeValue(value)
lengths[column] = max(
lengths[column],
len(
DUMP_REPLACEMENTS.get(getUnicode(value),
getUnicode(value))))
entries[column].append(value)
except KeyboardInterrupt:
kb.dumpKeyboardInterrupt = True
warnMsg = "user aborted during enumeration. sqlmap "
warnMsg += "will display partial output"
logger.warn(warnMsg)
except SqlmapConnectionException as ex:
errMsg = "connection exception detected ('%s'). sqlmap " % getSafeExString(
ex)
errMsg += "will display partial output"
logger.critical(errMsg)
return entries, lengths
def decodePage(page, contentEncoding, contentType, percentDecode=True):
"""
Decode compressed/charset HTTP response
>>> getText(decodePage(b"<html>foo&bar</html>", None, "text/html; charset=utf-8"))
'<html>foo&bar</html>'
"""
if not page or (conf.nullConnection and len(page) < 2):
return getUnicode(page)
if hasattr(contentEncoding, "lower"):
contentEncoding = contentEncoding.lower()
else:
contentEncoding = ""
if hasattr(contentType, "lower"):
contentType = contentType.lower()
else:
contentType = ""
if contentEncoding in ("gzip", "x-gzip", "deflate"):
if not kb.pageCompress:
return None
try:
if contentEncoding == "deflate":
data = io.BytesIO(
zlib.decompress(page, -15)
) # Reference: http://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations
else:
data = gzip.GzipFile("", "rb", 9, io.BytesIO(page))
size = struct.unpack(
"<l", page[-4:]
)[0] # Reference: http://pydoc.org/get.cgi/usr/local/lib/python2.5/gzip.py
if size > MAX_CONNECTION_TOTAL_SIZE:
raise Exception("size too large")
page = data.read()
except Exception as ex:
if "<html" not in page: # in some cases, invalid "Content-Encoding" appears for plain HTML (should be ignored)
errMsg = "detected invalid data for declared content "
errMsg += "encoding '%s' ('%s')" % (contentEncoding,
getSafeExString(ex))
singleTimeLogMessage(errMsg, logging.ERROR)
warnMsg = "turning off page compression"
singleTimeWarnMessage(warnMsg)
kb.pageCompress = False
raise SqlmapCompressionException
if not conf.encoding:
httpCharset, metaCharset = None, None
# Reference: http://stackoverflow.com/questions/1020892/python-urllib2-read-to-unicode
if contentType.find("charset=") != -1:
httpCharset = checkCharEncoding(contentType.split("charset=")[-1])
metaCharset = checkCharEncoding(
extractRegexResult(META_CHARSET_REGEX, page))
if (any((httpCharset, metaCharset)) and not all(
(httpCharset, metaCharset))) or (httpCharset == metaCharset
and all(
(httpCharset, metaCharset))):
kb.pageEncoding = httpCharset or metaCharset # Reference: http://bytes.com/topic/html-css/answers/154758-http-equiv-vs-true-header-has-precedence
debugMsg = "declared web page charset '%s'" % kb.pageEncoding
singleTimeLogMessage(debugMsg, logging.DEBUG, debugMsg)
else:
kb.pageEncoding = None
else:
kb.pageEncoding = conf.encoding
# can't do for all responses because we need to support binary files too
if isinstance(page, six.binary_type) and "text/" in contentType:
# e.g. 	Ãëàâà
if b"&#" in page:
page = re.sub(
b"&#x([0-9a-f]{1,2});", lambda _: decodeHex(
_.group(1)
if len(_.group(1)) == 2 else "0%s" % _.group(1)), page)
page = re.sub(
b"&#(\\d{1,3});", lambda _: six.int2byte(int(_.group(1)))
if int(_.group(1)) < 256 else _.group(0), page)
# e.g. %20%28%29
if percentDecode:
if b"%" in page:
page = re.sub(b"%([0-9a-fA-F]{2})",
lambda _: decodeHex(_.group(1)), page)
# e.g. &
page = re.sub(
b"&([^;]+);",
lambda _: six.int2byte(HTML_ENTITIES[getText(_.group(1))])
if HTML_ENTITIES.get(getText(_.group(1)), 256) < 256 else _.group(
0), page)
kb.pageEncoding = kb.pageEncoding or checkCharEncoding(
getHeuristicCharEncoding(page))
if (kb.pageEncoding or "").lower() == "utf-8-sig":
kb.pageEncoding = "utf-8"
if page and page.startswith(
"\xef\xbb\xbf"
): # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling)
page = page[3:]
page = getUnicode(page, kb.pageEncoding)
# e.g. ’…™
if "&#" in page:
def _(match):
retVal = match.group(0)
try:
retVal = _unichr(int(match.group(1)))
except (ValueError, OverflowError):
pass
return retVal
page = re.sub(r"&#(\d+);", _, page)
# e.g. ζ
page = re.sub(
r"&([^;]+);", lambda _: _unichr(HTML_ENTITIES[_.group(1)])
if HTML_ENTITIES.get(_.group(1), 0) > 255 else _.group(0), page)
return page
