def getRevisionNumber():
"""
Returns abbreviated commit hash number as retrieved with "git rev-parse --short HEAD"
>>> len(getRevisionNumber() or (' ' * 7)) == 7
True
"""
retVal = None
filePath = None
_ = os.path.dirname(__file__)
while True:
filePath = os.path.join(_, ".git", "HEAD")
if os.path.exists(filePath):
break
else:
filePath = None
if _ == os.path.dirname(_):
break
else:
_ = os.path.dirname(_)
while True:
if filePath and os.path.isfile(filePath):
with openFile(filePath, "r") as f:
content = f.read()
filePath = None
if content.startswith("ref: "):
filePath = os.path.join(_, ".git",
content.replace("ref: ",
"")).strip()
else:
match = re.match(r"(?i)[0-9a-f]{32}", content)
retVal = match.group(0) if match else None
break
else:
break
if not retVal:
try:
process = subprocess.Popen("git rev-parse --verify HEAD",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, _ = process.communicate()
match = re.search(r"(?i)[0-9a-f]{32}", getText(stdout or ""))
retVal = match.group(0) if match else None
except:
pass
return retVal[:7] if retVal else None
def decodePage(page, contentEncoding, contentType):
"""
Decode compressed/charset HTTP response
>>> getText(decodePage(b"<html>foo&bar</html>", None, "text/html; charset=utf-8"))
'<html>foo&bar</html>'
"""
if not page or (conf.nullConnection and len(page) < 2):
return getUnicode(page)
if hasattr(contentEncoding, "lower"):
contentEncoding = contentEncoding.lower()
else:
contentEncoding = ""
if hasattr(contentType, "lower"):
contentType = contentType.lower()
else:
contentType = ""
if contentEncoding in ("gzip", "x-gzip", "deflate"):
if not kb.pageCompress:
return None
try:
if contentEncoding == "deflate":
data = io.BytesIO(
zlib.decompress(page, -15)
) # Reference: http://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations
else:
data = gzip.GzipFile("", "rb", 9, io.BytesIO(page))
size = struct.unpack(
"<l", page[-4:]
)[0] # Reference: http://pydoc.org/get.cgi/usr/local/lib/python2.5/gzip.py
if size > MAX_CONNECTION_TOTAL_SIZE:
raise Exception("size too large")
page = data.read()
except Exception as ex:
if "<html" not in page: # in some cases, invalid "Content-Encoding" appears for plain HTML (should be ignored)
errMsg = "detected invalid data for declared content "
errMsg += "encoding '%s' ('%s')" % (contentEncoding,
getSafeExString(ex))
singleTimeLogMessage(errMsg, logging.ERROR)
warnMsg = "turning off page compression"
singleTimeWarnMessage(warnMsg)
kb.pageCompress = False
raise SqlmapCompressionException
if not conf.encoding:
httpCharset, metaCharset = None, None
# Reference: http://stackoverflow.com/questions/1020892/python-urllib2-read-to-unicode
if contentType.find("charset=") != -1:
httpCharset = checkCharEncoding(contentType.split("charset=")[-1])
metaCharset = checkCharEncoding(
extractRegexResult(META_CHARSET_REGEX, page))
if (any((httpCharset, metaCharset)) and not all(
(httpCharset, metaCharset))) or (httpCharset == metaCharset
and all(
(httpCharset, metaCharset))):
kb.pageEncoding = httpCharset or metaCharset # Reference: http://bytes.com/topic/html-css/answers/154758-http-equiv-vs-true-header-has-precedence
debugMsg = "declared web page charset '%s'" % kb.pageEncoding
singleTimeLogMessage(debugMsg, logging.DEBUG, debugMsg)
else:
kb.pageEncoding = None
else:
kb.pageEncoding = conf.encoding
# can't do for all responses because we need to support binary files too
if isinstance(page, six.binary_type) and "text/" in contentType:
# e.g. 	Ãëàâà
if b"&#" in page:
page = re.sub(
b"&#x([0-9a-f]{1,2});", lambda _: decodeHex(
_.group(1)
if len(_.group(1)) == 2 else "0%s" % _.group(1)), page)
page = re.sub(
b"&#(\\d{1,3});", lambda _: six.int2byte(int(_.group(1)))
if int(_.group(1)) < 256 else _.group(0), page)
# e.g. %20%28%29
if b"%" in page:
page = re.sub(b"%([0-9a-fA-F]{2})",
lambda _: decodeHex(_.group(1)), page)
# e.g. &
page = re.sub(
b"&([^;]+);",
lambda _: six.int2byte(htmlEntities[getText(_.group(1))])
if htmlEntities.get(getText(_.group(1)), 256) < 256 else _.group(
0), page)
kb.pageEncoding = kb.pageEncoding or checkCharEncoding(
getHeuristicCharEncoding(page))
if (kb.pageEncoding or "").lower() == "utf-8-sig":
kb.pageEncoding = "utf-8"
if page and page.startswith(
"\xef\xbb\xbf"
): # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling)
page = page[3:]
page = getUnicode(page, kb.pageEncoding)
# e.g. ’…™
if "&#" in page:
def _(match):
retVal = match.group(0)
try:
retVal = six.unichr(int(match.group(1)))
except (ValueError, OverflowError):
pass
return retVal
page = re.sub(r"&#(\d+);", _, page)
# e.g. ζ
page = re.sub(
r"&([^;]+);", lambda _: six.unichr(htmlEntities[_.group(1)])
if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)
return page
def update():
if not conf.updateAll:
return
success = False
if not os.path.exists(os.path.join(paths.SQLMAP_ROOT_PATH, ".git")):
warnMsg = "not a git repository. It is recommended to clone the 'sqlmapproject/sqlmap' repository "
warnMsg += "from GitHub (e.g. 'git clone --depth 1 %s sqlmap')" % GIT_REPOSITORY
logger.warn(warnMsg)
if VERSION == getLatestRevision():
logger.info("already at the latest revision '%s'" %
getRevisionNumber())
return
message = "do you want to try to fetch the latest 'zipball' from repository and extract it (experimental) ? [y/N]"
if readInput(message, default='N', boolean=True):
directory = os.path.abspath(paths.SQLMAP_ROOT_PATH)
try:
open(os.path.join(directory, "sqlmap.py"), "w+b")
except Exception as ex:
errMsg = "unable to update content of directory '%s' ('%s')" % (
directory, getSafeExString(ex))
logger.error(errMsg)
else:
attrs = os.stat(os.path.join(directory, "sqlmap.py")).st_mode
for wildcard in ('*', ".*"):
for _ in glob.glob(os.path.join(directory, wildcard)):
try:
if os.path.isdir(_):
shutil.rmtree(_)
else:
os.remove(_)
except:
pass
if glob.glob(os.path.join(directory, '*')):
errMsg = "unable to clear the content of directory '%s'" % directory
logger.error(errMsg)
else:
try:
archive = _urllib.request.urlretrieve(ZIPBALL_PAGE)[0]
with zipfile.ZipFile(archive) as f:
for info in f.infolist():
info.filename = re.sub(r"\Asqlmap[^/]+", "",
info.filename)
if info.filename:
f.extract(info, directory)
filepath = os.path.join(paths.SQLMAP_ROOT_PATH, "lib",
"core", "settings.py")
if os.path.isfile(filepath):
with openFile(filepath, "rb") as f:
version = re.search(
r"(?m)^VERSION\s*=\s*['\"]([^'\"]+)",
f.read()).group(1)
logger.info(
"updated to the latest version '%s#dev'" %
version)
success = True
except Exception as ex:
logger.error("update could not be completed ('%s')" %
getSafeExString(ex))
else:
if not success:
logger.error("update could not be completed")
else:
try:
os.chmod(os.path.join(directory, "sqlmap.py"),
attrs)
except OSError:
logger.warning(
"could not set the file attributes of '%s'"
% os.path.join(directory, "sqlmap.py"))
else:
infoMsg = "updating sqlmap to the latest development revision from the "
infoMsg += "GitHub repository"
logger.info(infoMsg)
debugMsg = "sqlmap will try to update itself using 'git' command"
logger.debug(debugMsg)
dataToStdout("\r[%s] [INFO] update in progress" % time.strftime("%X"))
try:
process = subprocess.Popen("git checkout . && git pull %s HEAD" %
GIT_REPOSITORY,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=paths.SQLMAP_ROOT_PATH)
pollProcess(process, True)
output, _ = process.communicate()
success = not process.returncode
except Exception as ex:
success = False
output = getSafeExString(ex)
finally:
output = getText(output)
if success:
logger.info("%s the latest revision '%s'" %
("already at" if "Already" in output else "updated to",
getRevisionNumber()))
else:
if "Not a git repository" in output:
errMsg = "not a valid git repository. Please checkout the 'sqlmapproject/sqlmap' repository "
errMsg += "from GitHub (e.g. 'git clone --depth 1 %s sqlmap')" % GIT_REPOSITORY
logger.error(errMsg)
else:
logger.error("update could not be completed ('%s')" %
re.sub(r"\W+", " ", output).strip())
if not success:
if IS_WIN:
infoMsg = "for Windows platform it's recommended "
infoMsg += "to use a GitHub for Windows client for updating "
infoMsg += "purposes (http://windows.github.com/) or just "
infoMsg += "download the latest snapshot from "
infoMsg += "https://github.com/sqlmapproject/sqlmap/downloads"
else:
infoMsg = "for Linux platform it's recommended "
infoMsg += "to install a standard 'git' package (e.g.: 'sudo apt-get install git')"
logger.info(infoMsg)
def dbTableValues(self, tableValues):
replication = None
rtable = None
dumpFP = None
appendToFile = False
warnFile = False
if tableValues is None:
return
db = tableValues["__infos__"]["db"]
if not db:
db = "All"
table = tableValues["__infos__"]["table"]
if conf.api:
self._write(tableValues, content_type=CONTENT_TYPE.DUMP_TABLE)
return
dumpDbPath = os.path.join(conf.dumpPath,
unsafeSQLIdentificatorNaming(db))
if conf.dumpFormat == DUMP_FORMAT.SQLITE:
replication = Replication(
os.path.join(conf.dumpPath,
"%s.sqlite3" % unsafeSQLIdentificatorNaming(db)))
elif conf.dumpFormat in (DUMP_FORMAT.CSV, DUMP_FORMAT.HTML):
if not os.path.isdir(dumpDbPath):
try:
os.makedirs(dumpDbPath)
except:
warnFile = True
_ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT,
unsafeSQLIdentificatorNaming(db))
dumpDbPath = os.path.join(
conf.dumpPath, "%s-%s" %
(_, hashlib.md5(getBytes(db)).hexdigest()[:8]))
if not os.path.isdir(dumpDbPath):
try:
os.makedirs(dumpDbPath)
except Exception as ex:
tempDir = tempfile.mkdtemp(prefix="sqlmapdb")
warnMsg = "unable to create dump directory "
warnMsg += "'%s' (%s). " % (dumpDbPath,
getSafeExString(ex))
warnMsg += "Using temporary directory '%s' instead" % tempDir
logger.warn(warnMsg)
dumpDbPath = tempDir
dumpFileName = os.path.join(
dumpDbPath,
re.sub(
r'[\\/]', UNSAFE_DUMP_FILEPATH_REPLACEMENT,
"%s.%s" % (unsafeSQLIdentificatorNaming(table),
conf.dumpFormat.lower())))
if not checkFile(dumpFileName, False):
try:
openFile(dumpFileName, "w+b").close()
except SqlmapSystemException:
raise
except:
warnFile = True
_ = re.sub(
r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT,
normalizeUnicode(unsafeSQLIdentificatorNaming(table)))
if len(_) < len(table) or IS_WIN and table.upper(
) in WINDOWS_RESERVED_NAMES:
_ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT,
unsafeSQLIdentificatorNaming(table))
dumpFileName = os.path.join(
dumpDbPath, "%s-%s.%s" %
(_, hashlib.md5(getBytes(table)).hexdigest()[:8],
conf.dumpFormat.lower()))
else:
dumpFileName = os.path.join(
dumpDbPath, "%s.%s" % (_, conf.dumpFormat.lower()))
else:
appendToFile = any((conf.limitStart, conf.limitStop))
if not appendToFile:
count = 1
while True:
candidate = "%s.%d" % (dumpFileName, count)
if not checkFile(candidate, False):
try:
shutil.copyfile(dumpFileName, candidate)
except IOError:
pass
finally:
break
else:
count += 1
dumpFP = openFile(dumpFileName,
"wb" if not appendToFile else "ab",
buffering=DUMP_FILE_BUFFER_SIZE)
count = int(tableValues["__infos__"]["count"])
separator = str()
field = 1
fields = len(tableValues) - 1
columns = prioritySortColumns(list(tableValues.keys()))
if conf.col:
cols = conf.col.split(',')
columns = sorted(columns,
key=lambda _: cols.index(_) if _ in cols else 0)
for column in columns:
if column != "__infos__":
info = tableValues[column]
lines = "-" * (int(info["length"]) + 2)
separator += "+%s" % lines
separator += "+"
self._write(
"Database: %s\nTable: %s" %
(unsafeSQLIdentificatorNaming(db) if db else "Current database",
unsafeSQLIdentificatorNaming(table)))
if conf.dumpFormat == DUMP_FORMAT.SQLITE:
cols = []
for column in columns:
if column != "__infos__":
colType = Replication.INTEGER
for value in tableValues[column]['values']:
try:
if not value or value == " ": # NULL
continue
int(value)
except ValueError:
colType = None
break
if colType is None:
colType = Replication.REAL
for value in tableValues[column]['values']:
try:
if not value or value == " ": # NULL
continue
float(value)
except ValueError:
colType = None
break
cols.append((unsafeSQLIdentificatorNaming(column),
colType if colType else Replication.TEXT))
rtable = replication.createTable(table, cols)
elif conf.dumpFormat == DUMP_FORMAT.HTML:
dataToDumpFile(dumpFP, "<!DOCTYPE html>\n<html>\n<head>\n")
dataToDumpFile(
dumpFP,
"<meta http-equiv=\"Content-type\" content=\"text/html;charset=%s\">\n"
% UNICODE_ENCODING)
dataToDumpFile(
dumpFP, "<meta name=\"generator\" content=\"%s\" />\n" %
VERSION_STRING)
dataToDumpFile(
dumpFP, "<title>%s</title>\n" %
("%s%s" %
("%s." % db if METADB_SUFFIX not in db else "", table)))
dataToDumpFile(dumpFP, HTML_DUMP_CSS_STYLE)
dataToDumpFile(dumpFP,
"\n</head>\n<body>\n<table>\n<thead>\n<tr>\n")
if count == 1:
self._write("[1 entry]")
else:
self._write("[%d entries]" % count)
self._write(separator)
for column in columns:
if column != "__infos__":
info = tableValues[column]
column = unsafeSQLIdentificatorNaming(column)
maxlength = int(info["length"])
blank = " " * (maxlength - len(column))
self._write("| %s%s" % (column, blank), newline=False)
if not appendToFile:
if conf.dumpFormat == DUMP_FORMAT.CSV:
if field == fields:
dataToDumpFile(dumpFP, "%s" % safeCSValue(column))
else:
dataToDumpFile(
dumpFP,
"%s%s" % (safeCSValue(column), conf.csvDel))
elif conf.dumpFormat == DUMP_FORMAT.HTML:
dataToDumpFile(
dumpFP, "<th>%s</th>" % getUnicode(
cgi.escape(column).encode(
"ascii", "xmlcharrefreplace")))
field += 1
if conf.dumpFormat == DUMP_FORMAT.HTML:
dataToDumpFile(dumpFP, "\n</tr>\n</thead>\n<tbody>\n")
self._write("|\n%s" % separator)
if conf.dumpFormat == DUMP_FORMAT.CSV:
dataToDumpFile(dumpFP, "\n" if not appendToFile else "")
elif conf.dumpFormat == DUMP_FORMAT.SQLITE:
rtable.beginTransaction()
if count > TRIM_STDOUT_DUMP_SIZE:
warnMsg = "console output will be trimmed to "
warnMsg += "last %d rows due to " % TRIM_STDOUT_DUMP_SIZE
warnMsg += "large table size"
logger.warning(warnMsg)
for i in xrange(count):
console = (i >= count - TRIM_STDOUT_DUMP_SIZE)
field = 1
values = []
if conf.dumpFormat == DUMP_FORMAT.HTML:
dataToDumpFile(dumpFP, "<tr>")
for column in columns:
if column != "__infos__":
info = tableValues[column]
if len(info["values"]) <= i:
continue
if info["values"][i] is None:
value = u''
else:
value = getUnicode(info["values"][i])
value = DUMP_REPLACEMENTS.get(value, value)
values.append(value)
maxlength = int(info["length"])
blank = " " * (maxlength - len(value))
self._write("| %s%s" % (value, blank),
newline=False,
console=console)
if len(value
) > MIN_BINARY_DISK_DUMP_SIZE and r'\x' in value:
try:
mimetype = getText(
magic.from_buffer(value, mime=True))
if any(
mimetype.startswith(_)
for _ in ("application", "image")):
if not os.path.isdir(dumpDbPath):
os.makedirs(dumpDbPath)
_ = re.sub(
r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT,
normalizeUnicode(
unsafeSQLIdentificatorNaming(column)))
filepath = os.path.join(
dumpDbPath,
"%s-%d.bin" % (_, randomInt(8)))
warnMsg = "writing binary ('%s') content to file '%s' " % (
mimetype, filepath)
logger.warn(warnMsg)
with open(filepath, "wb") as f:
_ = safechardecode(value, True)
f.write(_)
except magic.MagicException as ex:
logger.debug(getSafeExString(ex))
if conf.dumpFormat == DUMP_FORMAT.CSV:
if field == fields:
dataToDumpFile(dumpFP, "%s" % safeCSValue(value))
else:
dataToDumpFile(
dumpFP,
"%s%s" % (safeCSValue(value), conf.csvDel))
elif conf.dumpFormat == DUMP_FORMAT.HTML:
dataToDumpFile(
dumpFP, "<td>%s</td>" % getUnicode(
cgi.escape(value).encode(
"ascii", "xmlcharrefreplace")))
field += 1
if conf.dumpFormat == DUMP_FORMAT.SQLITE:
try:
rtable.insert(values)
except SqlmapValueException:
pass
elif conf.dumpFormat == DUMP_FORMAT.CSV:
dataToDumpFile(dumpFP, "\n")
elif conf.dumpFormat == DUMP_FORMAT.HTML:
dataToDumpFile(dumpFP, "</tr>\n")
self._write("|", console=console)
self._write("%s\n" % separator)
if conf.dumpFormat == DUMP_FORMAT.SQLITE:
rtable.endTransaction()
logger.info("table '%s.%s' dumped to sqlite3 database '%s'" %
(db, table, replication.dbpath))
elif conf.dumpFormat in (DUMP_FORMAT.CSV, DUMP_FORMAT.HTML):
if conf.dumpFormat == DUMP_FORMAT.HTML:
dataToDumpFile(dumpFP, "</tbody>\n</table>\n</body>\n</html>")
else:
dataToDumpFile(dumpFP, "\n")
dumpFP.close()
msg = "table '%s.%s' dumped to %s file '%s'" % (
db, table, conf.dumpFormat, dumpFileName)
if not warnFile:
logger.info(msg)
else:
logger.warn(msg)