def encodeHex(value, binary=True):
"""
Returns a encoded representation of provided string value
>>> encodeHex(b"123") == b"313233"
True
>>> encodeHex("123", binary=False)
'313233'
>>> encodeHex(b"123"[0]) == b"31"
True
"""
if isinstance(value, int):
value = six.unichr(value)
if isinstance(value, six.text_type):
value = value.encode(UNICODE_ENCODING)
try:
retVal = codecs.encode(value, "hex")
except LookupError:
retVal = binascii.hexlify(value)
if not binary:
retVal = getText(retVal)
return retVal
def _(match):
retVal = match.group(0)
try:
retVal = six.unichr(int(match.group(1)))
except (ValueError, OverflowError):
pass
return retVal
def getBytes(value, encoding=UNICODE_ENCODING, errors="strict", unsafe=True):
"""
Returns byte representation of provided Unicode value
>>> getBytes(u"foo\\\\x01\\\\x83\\\\xffbar") == b"foo\\x01\\x83\\xffbar"
True
"""
retVal = value
if isinstance(value, six.text_type):
if INVALID_UNICODE_PRIVATE_AREA:
if unsafe:
for char in xrange(0xF0000, 0xF00FF + 1):
value = value.replace(
six.unichr(char),
"%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000))
retVal = value.encode(encoding, errors)
if unsafe:
retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER,
lambda _: decodeHex(_.group(1)), retVal)
else:
retVal = value.encode(encoding, errors)
if unsafe:
retVal = re.sub(b"\\\\x([0-9a-f]{2})",
lambda _: decodeHex(_.group(1)), retVal)
return retVal
def process(current):
if not current:
return ""
if not any(current[_] for _ in current):
if len(current) > 1:
items = []
previous = None
start = None
for _ in sorted(current) + [six.unichr(65535)]:
if previous is not None:
if ord(_) == ord(previous) + 1:
pass
else:
if start != previous:
if start == '0' and previous == '9':
items.append(r"\d")
else:
items.append("%s-%s" % (re.escape(start), re.escape(previous)))
else:
items.append(re.escape(previous))
start = _
if start is None:
start = _
previous = _
return ("[%s]" % "".join(items)) if len(items) > 1 or '-' in items[0] else "".join(items)
else:
return re.escape(list(current.keys())[0])
else:
return ("(?:%s)" if len(current) > 1 else "%s") % ('|'.join("%s%s" % (re.escape(_), process(current[_])) for _ in sorted(current))).replace('|'.join(str(_) for _ in xrange(10)), r"\d")
def _reversible(ex):
if isinstance(ex, UnicodeDecodeError):
if INVALID_UNICODE_PRIVATE_AREA:
return (u"".join(
six.unichr(
int('000f00%2x' %
(_ if isinstance(_, int) else ord(_)), 16))
for _ in ex.object[ex.start:ex.end]), ex.end)
else:
return (u"".join(INVALID_UNICODE_CHAR_FORMAT %
(_ if isinstance(_, int) else ord(_))
for _ in ex.object[ex.start:ex.end]), ex.end)
def htmlunescape(value):
"""
Returns (basic conversion) HTML unescaped value
>>> htmlunescape('a<b')
'a<b'
"""
retVal = value
if value and isinstance(value, six.string_types):
replacements = (("<", '<'), (">", '>'), (""", '"'), (" ", ' '), ("&", '&'), ("'", "'"))
for code, value in replacements:
retVal = retVal.replace(code, value)
try:
retVal = re.sub(r"&#x([^ ;]+);", lambda match: six.unichr(int(match.group(1), 16)), retVal)
except ValueError:
pass
return retVal
def decodePage(page, contentEncoding, contentType):
"""
Decode compressed/charset HTTP response
>>> getText(decodePage(b"<html>foo&bar</html>", None, "text/html; charset=utf-8"))
'<html>foo&bar</html>'
"""
if not page or (conf.nullConnection and len(page) < 2):
return getUnicode(page)
if hasattr(contentEncoding, "lower"):
contentEncoding = contentEncoding.lower()
else:
contentEncoding = ""
if hasattr(contentType, "lower"):
contentType = contentType.lower()
else:
contentType = ""
if contentEncoding in ("gzip", "x-gzip", "deflate"):
if not kb.pageCompress:
return None
try:
if contentEncoding == "deflate":
data = io.BytesIO(
zlib.decompress(page, -15)
) # Reference: http://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations
else:
data = gzip.GzipFile("", "rb", 9, io.BytesIO(page))
size = struct.unpack(
"<l", page[-4:]
)[0] # Reference: http://pydoc.org/get.cgi/usr/local/lib/python2.5/gzip.py
if size > MAX_CONNECTION_TOTAL_SIZE:
raise Exception("size too large")
page = data.read()
except Exception as ex:
if "<html" not in page: # in some cases, invalid "Content-Encoding" appears for plain HTML (should be ignored)
errMsg = "detected invalid data for declared content "
errMsg += "encoding '%s' ('%s')" % (contentEncoding,
getSafeExString(ex))
singleTimeLogMessage(errMsg, logging.ERROR)
warnMsg = "turning off page compression"
singleTimeWarnMessage(warnMsg)
kb.pageCompress = False
raise SqlmapCompressionException
if not conf.encoding:
httpCharset, metaCharset = None, None
# Reference: http://stackoverflow.com/questions/1020892/python-urllib2-read-to-unicode
if contentType.find("charset=") != -1:
httpCharset = checkCharEncoding(contentType.split("charset=")[-1])
metaCharset = checkCharEncoding(
extractRegexResult(META_CHARSET_REGEX, page))
if (any((httpCharset, metaCharset)) and not all(
(httpCharset, metaCharset))) or (httpCharset == metaCharset
and all(
(httpCharset, metaCharset))):
kb.pageEncoding = httpCharset or metaCharset # Reference: http://bytes.com/topic/html-css/answers/154758-http-equiv-vs-true-header-has-precedence
debugMsg = "declared web page charset '%s'" % kb.pageEncoding
singleTimeLogMessage(debugMsg, logging.DEBUG, debugMsg)
else:
kb.pageEncoding = None
else:
kb.pageEncoding = conf.encoding
# can't do for all responses because we need to support binary files too
if isinstance(page, six.binary_type) and "text/" in contentType:
# e.g. 	Ãëàâà
if b"&#" in page:
page = re.sub(
b"&#x([0-9a-f]{1,2});", lambda _: decodeHex(
_.group(1)
if len(_.group(1)) == 2 else "0%s" % _.group(1)), page)
page = re.sub(
b"&#(\\d{1,3});", lambda _: six.int2byte(int(_.group(1)))
if int(_.group(1)) < 256 else _.group(0), page)
# e.g. %20%28%29
if b"%" in page:
page = re.sub(b"%([0-9a-fA-F]{2})",
lambda _: decodeHex(_.group(1)), page)
# e.g. &
page = re.sub(
b"&([^;]+);",
lambda _: six.int2byte(htmlEntities[getText(_.group(1))])
if htmlEntities.get(getText(_.group(1)), 256) < 256 else _.group(
0), page)
kb.pageEncoding = kb.pageEncoding or checkCharEncoding(
getHeuristicCharEncoding(page))
if (kb.pageEncoding or "").lower() == "utf-8-sig":
kb.pageEncoding = "utf-8"
if page and page.startswith(
"\xef\xbb\xbf"
): # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling)
page = page[3:]
page = getUnicode(page, kb.pageEncoding)
# e.g. ’…™
if "&#" in page:
def _(match):
retVal = match.group(0)
try:
retVal = six.unichr(int(match.group(1)))
except (ValueError, OverflowError):
pass
return retVal
page = re.sub(r"&#(\d+);", _, page)
# e.g. ζ
page = re.sub(
r"&([^;]+);", lambda _: six.unichr(htmlEntities[_.group(1)])
if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)
return page
def pivotDumpTable(table, colList, count=None, blind=True, alias=None):
lengths = {}
entries = {}
dumpNode = queries[Backend.getIdentifiedDbms()].dump_table.blind
validColumnList = False
validPivotValue = False
if count is None:
query = dumpNode.count % table
query = agent.whereQuery(query)
count = inject.getValue(
query,
union=False,
error=False,
expected=EXPECTED.INT,
charsetType=CHARSET_TYPE.DIGITS) if blind else inject.getValue(
query, blind=False, time=False, expected=EXPECTED.INT)
if hasattr(count, "isdigit") and count.isdigit():
count = int(count)
if count == 0:
infoMsg = "table '%s' appears to be empty" % unsafeSQLIdentificatorNaming(
table)
logger.info(infoMsg)
for column in colList:
lengths[column] = len(column)
entries[column] = []
return entries, lengths
elif not isNumPosStrValue(count):
return None
for column in colList:
lengths[column] = 0
entries[column] = BigArray()
colList = filterNone(
sorted(colList, key=lambda x: len(x) if x else MAX_INT))
if conf.pivotColumn:
for _ in colList:
if re.search(r"(.+\.)?%s" % re.escape(conf.pivotColumn), _, re.I):
infoMsg = "using column '%s' as a pivot " % conf.pivotColumn
infoMsg += "for retrieving row data"
logger.info(infoMsg)
colList.remove(_)
colList.insert(0, _)
validPivotValue = True
break
if not validPivotValue:
warnMsg = "column '%s' not " % conf.pivotColumn
warnMsg += "found in table '%s'" % table
logger.warn(warnMsg)
if not validPivotValue:
for column in colList:
infoMsg = "fetching number of distinct "
infoMsg += "values for column '%s'" % column.replace(
("%s." % alias) if alias else "", "")
logger.info(infoMsg)
query = dumpNode.count2 % (column, table)
query = agent.whereQuery(query)
value = inject.getValue(query,
blind=blind,
union=not blind,
error=not blind,
expected=EXPECTED.INT,
charsetType=CHARSET_TYPE.DIGITS)
if isNumPosStrValue(value):
validColumnList = True
if value == count:
infoMsg = "using column '%s' as a pivot " % column.replace(
("%s." % alias) if alias else "", "")
infoMsg += "for retrieving row data"
logger.info(infoMsg)
validPivotValue = True
colList.remove(column)
colList.insert(0, column)
break
if not validColumnList:
errMsg = "all column name(s) provided are non-existent"
raise SqlmapNoneDataException(errMsg)
if not validPivotValue:
warnMsg = "no proper pivot column provided (with unique values)."
warnMsg += " It won't be possible to retrieve all rows"
logger.warn(warnMsg)
pivotValue = " "
breakRetrieval = False
def _(column, pivotValue):
if column == colList[0]:
query = dumpNode.query.replace(
"'%s'" if unescaper.escape(pivotValue, False) != pivotValue
else "%s", "%s") % (agent.preprocessField(
table, column), table, agent.preprocessField(
table, column), unescaper.escape(pivotValue, False))
else:
query = dumpNode.query2.replace(
"'%s'"
if unescaper.escape(pivotValue, False) != pivotValue else "%s",
"%s") % (agent.preprocessField(table, column), table,
agent.preprocessField(table, colList[0]),
unescaper.escape(pivotValue, False))
query = agent.whereQuery(query)
return unArrayizeValue(
inject.getValue(query,
blind=blind,
time=blind,
union=not blind,
error=not blind))
try:
for i in xrange(count):
if breakRetrieval:
break
for column in colList:
value = _(column, pivotValue)
if column == colList[0]:
if isNoneValue(value):
try:
for pivotValue in filterNone(
(" " if pivotValue == " " else None,
"%s%s" % (pivotValue[0],
six.unichr(ord(pivotValue[1]) + 1))
if len(pivotValue) > 1 else None,
six.unichr(ord(pivotValue[0]) + 1))):
value = _(column, pivotValue)
if not isNoneValue(value):
break
except ValueError:
pass
if isNoneValue(value) or value == NULL:
breakRetrieval = True
break
pivotValue = safechardecode(value)
if conf.limitStart or conf.limitStop:
if conf.limitStart and (i + 1) < conf.limitStart:
warnMsg = "skipping first %d pivot " % conf.limitStart
warnMsg += "point values"
singleTimeWarnMessage(warnMsg)
break
elif conf.limitStop and (i + 1) > conf.limitStop:
breakRetrieval = True
break
value = "" if isNoneValue(value) else unArrayizeValue(value)
lengths[column] = max(
lengths[column],
len(
DUMP_REPLACEMENTS.get(getUnicode(value),
getUnicode(value))))
entries[column].append(value)
except KeyboardInterrupt:
kb.dumpKeyboardInterrupt = True
warnMsg = "user aborted during enumeration. sqlmap "
warnMsg += "will display partial output"
logger.warn(warnMsg)
except SqlmapConnectionException as ex:
errMsg = "connection exception detected ('%s'). sqlmap " % getSafeExString(
ex)
errMsg += "will display partial output"
logger.critical(errMsg)
return entries, lengths
