def getRevisionNumber(): """ Returns abbreviated commit hash number as retrieved with "git rev-parse --short HEAD" >>> len(getRevisionNumber() or (' ' * 7)) == 7 True """ retVal = None filePath = None _ = os.path.dirname(__file__) while True: filePath = os.path.join(_, ".git", "HEAD") if os.path.exists(filePath): break else: filePath = None if _ == os.path.dirname(_): break else: _ = os.path.dirname(_) while True: if filePath and os.path.isfile(filePath): with openFile(filePath, "r") as f: content = f.read() filePath = None if content.startswith("ref: "): filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip() else: match = re.match(r"(?i)[0-9a-f]{32}", content) retVal = match.group(0) if match else None break else: break if not retVal: try: process = subprocess.Popen("git rev-parse --verify HEAD", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, _ = process.communicate() match = re.search(r"(?i)[0-9a-f]{32}", getText(stdout or "")) retVal = match.group(0) if match else None except: pass return retVal[:7] if retVal else None
def decodePage(page, contentEncoding, contentType): """ Decode compressed/charset HTTP response >>> getText(decodePage(b"<html>foo&bar</html>", None, "text/html; charset=utf-8")) '<html>foo&bar</html>' """ if not page or (conf.nullConnection and len(page) < 2): return getUnicode(page) if hasattr(contentEncoding, "lower"): contentEncoding = contentEncoding.lower() else: contentEncoding = "" if hasattr(contentType, "lower"): contentType = contentType.lower() else: contentType = "" if contentEncoding in ("gzip", "x-gzip", "deflate"): if not kb.pageCompress: return None try: if contentEncoding == "deflate": data = io.BytesIO( zlib.decompress(page, -15) ) # Reference: http://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations else: data = gzip.GzipFile("", "rb", 9, io.BytesIO(page)) size = struct.unpack( "<l", page[-4:] )[0] # Reference: http://pydoc.org/get.cgi/usr/local/lib/python2.5/gzip.py if size > MAX_CONNECTION_TOTAL_SIZE: raise Exception("size too large") page = data.read() except Exception as ex: if "<html" not in page: # in some cases, invalid "Content-Encoding" appears for plain HTML (should be ignored) errMsg = "detected invalid data for declared content " errMsg += "encoding '%s' ('%s')" % (contentEncoding, getSafeExString(ex)) singleTimeLogMessage(errMsg, logging.ERROR) warnMsg = "turning off page compression" singleTimeWarnMessage(warnMsg) kb.pageCompress = False raise SqlmapCompressionException if not conf.encoding: httpCharset, metaCharset = None, None # Reference: http://stackoverflow.com/questions/1020892/python-urllib2-read-to-unicode if contentType.find("charset=") != -1: httpCharset = checkCharEncoding(contentType.split("charset=")[-1]) metaCharset = checkCharEncoding( extractRegexResult(META_CHARSET_REGEX, page)) if (any((httpCharset, metaCharset)) and not all( (httpCharset, metaCharset))) or (httpCharset == metaCharset and all( (httpCharset, metaCharset))): kb.pageEncoding = httpCharset or metaCharset # Reference: http://bytes.com/topic/html-css/answers/154758-http-equiv-vs-true-header-has-precedence debugMsg = "declared web page charset '%s'" % kb.pageEncoding singleTimeLogMessage(debugMsg, logging.DEBUG, debugMsg) else: kb.pageEncoding = None else: kb.pageEncoding = conf.encoding # can't do for all responses because we need to support binary files too if isinstance(page, six.binary_type) and "text/" in contentType: # e.g. 	Ãëàâà if b"&#" in page: page = re.sub( b"&#x([0-9a-f]{1,2});", lambda _: decodeHex( _.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)), page) page = re.sub( b"&#(\\d{1,3});", lambda _: six.int2byte(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page) # e.g. %20%28%29 if b"%" in page: page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page) # e.g. & page = re.sub( b"&([^;]+);", lambda _: six.int2byte(htmlEntities[getText(_.group(1))]) if htmlEntities.get(getText(_.group(1)), 256) < 256 else _.group( 0), page) kb.pageEncoding = kb.pageEncoding or checkCharEncoding( getHeuristicCharEncoding(page)) if (kb.pageEncoding or "").lower() == "utf-8-sig": kb.pageEncoding = "utf-8" if page and page.startswith( "\xef\xbb\xbf" ): # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling) page = page[3:] page = getUnicode(page, kb.pageEncoding) # e.g. ’…™ if "&#" in page: def _(match): retVal = match.group(0) try: retVal = six.unichr(int(match.group(1))) except (ValueError, OverflowError): pass return retVal page = re.sub(r"&#(\d+);", _, page) # e.g. ζ page = re.sub( r"&([^;]+);", lambda _: six.unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page) return page
def update(): if not conf.updateAll: return success = False if not os.path.exists(os.path.join(paths.SQLMAP_ROOT_PATH, ".git")): warnMsg = "not a git repository. It is recommended to clone the 'sqlmapproject/sqlmap' repository " warnMsg += "from GitHub (e.g. 'git clone --depth 1 %s sqlmap')" % GIT_REPOSITORY logger.warn(warnMsg) if VERSION == getLatestRevision(): logger.info("already at the latest revision '%s'" % getRevisionNumber()) return message = "do you want to try to fetch the latest 'zipball' from repository and extract it (experimental) ? [y/N]" if readInput(message, default='N', boolean=True): directory = os.path.abspath(paths.SQLMAP_ROOT_PATH) try: open(os.path.join(directory, "sqlmap.py"), "w+b") except Exception as ex: errMsg = "unable to update content of directory '%s' ('%s')" % ( directory, getSafeExString(ex)) logger.error(errMsg) else: attrs = os.stat(os.path.join(directory, "sqlmap.py")).st_mode for wildcard in ('*', ".*"): for _ in glob.glob(os.path.join(directory, wildcard)): try: if os.path.isdir(_): shutil.rmtree(_) else: os.remove(_) except: pass if glob.glob(os.path.join(directory, '*')): errMsg = "unable to clear the content of directory '%s'" % directory logger.error(errMsg) else: try: archive = _urllib.request.urlretrieve(ZIPBALL_PAGE)[0] with zipfile.ZipFile(archive) as f: for info in f.infolist(): info.filename = re.sub(r"\Asqlmap[^/]+", "", info.filename) if info.filename: f.extract(info, directory) filepath = os.path.join(paths.SQLMAP_ROOT_PATH, "lib", "core", "settings.py") if os.path.isfile(filepath): with openFile(filepath, "rb") as f: version = re.search( r"(?m)^VERSION\s*=\s*['\"]([^'\"]+)", f.read()).group(1) logger.info( "updated to the latest version '%s#dev'" % version) success = True except Exception as ex: logger.error("update could not be completed ('%s')" % getSafeExString(ex)) else: if not success: logger.error("update could not be completed") else: try: os.chmod(os.path.join(directory, "sqlmap.py"), attrs) except OSError: logger.warning( "could not set the file attributes of '%s'" % os.path.join(directory, "sqlmap.py")) else: infoMsg = "updating sqlmap to the latest development revision from the " infoMsg += "GitHub repository" logger.info(infoMsg) debugMsg = "sqlmap will try to update itself using 'git' command" logger.debug(debugMsg) dataToStdout("\r[%s] [INFO] update in progress" % time.strftime("%X")) try: process = subprocess.Popen("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=paths.SQLMAP_ROOT_PATH) pollProcess(process, True) output, _ = process.communicate() success = not process.returncode except Exception as ex: success = False output = getSafeExString(ex) finally: output = getText(output) if success: logger.info("%s the latest revision '%s'" % ("already at" if "Already" in output else "updated to", getRevisionNumber())) else: if "Not a git repository" in output: errMsg = "not a valid git repository. Please checkout the 'sqlmapproject/sqlmap' repository " errMsg += "from GitHub (e.g. 'git clone --depth 1 %s sqlmap')" % GIT_REPOSITORY logger.error(errMsg) else: logger.error("update could not be completed ('%s')" % re.sub(r"\W+", " ", output).strip()) if not success: if IS_WIN: infoMsg = "for Windows platform it's recommended " infoMsg += "to use a GitHub for Windows client for updating " infoMsg += "purposes (http://windows.github.com/) or just " infoMsg += "download the latest snapshot from " infoMsg += "https://github.com/sqlmapproject/sqlmap/downloads" else: infoMsg = "for Linux platform it's recommended " infoMsg += "to install a standard 'git' package (e.g.: 'sudo apt-get install git')" logger.info(infoMsg)
def dbTableValues(self, tableValues): replication = None rtable = None dumpFP = None appendToFile = False warnFile = False if tableValues is None: return db = tableValues["__infos__"]["db"] if not db: db = "All" table = tableValues["__infos__"]["table"] if conf.api: self._write(tableValues, content_type=CONTENT_TYPE.DUMP_TABLE) return dumpDbPath = os.path.join(conf.dumpPath, unsafeSQLIdentificatorNaming(db)) if conf.dumpFormat == DUMP_FORMAT.SQLITE: replication = Replication( os.path.join(conf.dumpPath, "%s.sqlite3" % unsafeSQLIdentificatorNaming(db))) elif conf.dumpFormat in (DUMP_FORMAT.CSV, DUMP_FORMAT.HTML): if not os.path.isdir(dumpDbPath): try: os.makedirs(dumpDbPath) except: warnFile = True _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(db)) dumpDbPath = os.path.join( conf.dumpPath, "%s-%s" % (_, hashlib.md5(getBytes(db)).hexdigest()[:8])) if not os.path.isdir(dumpDbPath): try: os.makedirs(dumpDbPath) except Exception as ex: tempDir = tempfile.mkdtemp(prefix="sqlmapdb") warnMsg = "unable to create dump directory " warnMsg += "'%s' (%s). " % (dumpDbPath, getSafeExString(ex)) warnMsg += "Using temporary directory '%s' instead" % tempDir logger.warn(warnMsg) dumpDbPath = tempDir dumpFileName = os.path.join( dumpDbPath, re.sub( r'[\\/]', UNSAFE_DUMP_FILEPATH_REPLACEMENT, "%s.%s" % (unsafeSQLIdentificatorNaming(table), conf.dumpFormat.lower()))) if not checkFile(dumpFileName, False): try: openFile(dumpFileName, "w+b").close() except SqlmapSystemException: raise except: warnFile = True _ = re.sub( r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, normalizeUnicode(unsafeSQLIdentificatorNaming(table))) if len(_) < len(table) or IS_WIN and table.upper( ) in WINDOWS_RESERVED_NAMES: _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(table)) dumpFileName = os.path.join( dumpDbPath, "%s-%s.%s" % (_, hashlib.md5(getBytes(table)).hexdigest()[:8], conf.dumpFormat.lower())) else: dumpFileName = os.path.join( dumpDbPath, "%s.%s" % (_, conf.dumpFormat.lower())) else: appendToFile = any((conf.limitStart, conf.limitStop)) if not appendToFile: count = 1 while True: candidate = "%s.%d" % (dumpFileName, count) if not checkFile(candidate, False): try: shutil.copyfile(dumpFileName, candidate) except IOError: pass finally: break else: count += 1 dumpFP = openFile(dumpFileName, "wb" if not appendToFile else "ab", buffering=DUMP_FILE_BUFFER_SIZE) count = int(tableValues["__infos__"]["count"]) separator = str() field = 1 fields = len(tableValues) - 1 columns = prioritySortColumns(list(tableValues.keys())) if conf.col: cols = conf.col.split(',') columns = sorted(columns, key=lambda _: cols.index(_) if _ in cols else 0) for column in columns: if column != "__infos__": info = tableValues[column] lines = "-" * (int(info["length"]) + 2) separator += "+%s" % lines separator += "+" self._write( "Database: %s\nTable: %s" % (unsafeSQLIdentificatorNaming(db) if db else "Current database", unsafeSQLIdentificatorNaming(table))) if conf.dumpFormat == DUMP_FORMAT.SQLITE: cols = [] for column in columns: if column != "__infos__": colType = Replication.INTEGER for value in tableValues[column]['values']: try: if not value or value == " ": # NULL continue int(value) except ValueError: colType = None break if colType is None: colType = Replication.REAL for value in tableValues[column]['values']: try: if not value or value == " ": # NULL continue float(value) except ValueError: colType = None break cols.append((unsafeSQLIdentificatorNaming(column), colType if colType else Replication.TEXT)) rtable = replication.createTable(table, cols) elif conf.dumpFormat == DUMP_FORMAT.HTML: dataToDumpFile(dumpFP, "<!DOCTYPE html>\n<html>\n<head>\n") dataToDumpFile( dumpFP, "<meta http-equiv=\"Content-type\" content=\"text/html;charset=%s\">\n" % UNICODE_ENCODING) dataToDumpFile( dumpFP, "<meta name=\"generator\" content=\"%s\" />\n" % VERSION_STRING) dataToDumpFile( dumpFP, "<title>%s</title>\n" % ("%s%s" % ("%s." % db if METADB_SUFFIX not in db else "", table))) dataToDumpFile(dumpFP, HTML_DUMP_CSS_STYLE) dataToDumpFile(dumpFP, "\n</head>\n<body>\n<table>\n<thead>\n<tr>\n") if count == 1: self._write("[1 entry]") else: self._write("[%d entries]" % count) self._write(separator) for column in columns: if column != "__infos__": info = tableValues[column] column = unsafeSQLIdentificatorNaming(column) maxlength = int(info["length"]) blank = " " * (maxlength - len(column)) self._write("| %s%s" % (column, blank), newline=False) if not appendToFile: if conf.dumpFormat == DUMP_FORMAT.CSV: if field == fields: dataToDumpFile(dumpFP, "%s" % safeCSValue(column)) else: dataToDumpFile( dumpFP, "%s%s" % (safeCSValue(column), conf.csvDel)) elif conf.dumpFormat == DUMP_FORMAT.HTML: dataToDumpFile( dumpFP, "<th>%s</th>" % getUnicode( cgi.escape(column).encode( "ascii", "xmlcharrefreplace"))) field += 1 if conf.dumpFormat == DUMP_FORMAT.HTML: dataToDumpFile(dumpFP, "\n</tr>\n</thead>\n<tbody>\n") self._write("|\n%s" % separator) if conf.dumpFormat == DUMP_FORMAT.CSV: dataToDumpFile(dumpFP, "\n" if not appendToFile else "") elif conf.dumpFormat == DUMP_FORMAT.SQLITE: rtable.beginTransaction() if count > TRIM_STDOUT_DUMP_SIZE: warnMsg = "console output will be trimmed to " warnMsg += "last %d rows due to " % TRIM_STDOUT_DUMP_SIZE warnMsg += "large table size" logger.warning(warnMsg) for i in xrange(count): console = (i >= count - TRIM_STDOUT_DUMP_SIZE) field = 1 values = [] if conf.dumpFormat == DUMP_FORMAT.HTML: dataToDumpFile(dumpFP, "<tr>") for column in columns: if column != "__infos__": info = tableValues[column] if len(info["values"]) <= i: continue if info["values"][i] is None: value = u'' else: value = getUnicode(info["values"][i]) value = DUMP_REPLACEMENTS.get(value, value) values.append(value) maxlength = int(info["length"]) blank = " " * (maxlength - len(value)) self._write("| %s%s" % (value, blank), newline=False, console=console) if len(value ) > MIN_BINARY_DISK_DUMP_SIZE and r'\x' in value: try: mimetype = getText( magic.from_buffer(value, mime=True)) if any( mimetype.startswith(_) for _ in ("application", "image")): if not os.path.isdir(dumpDbPath): os.makedirs(dumpDbPath) _ = re.sub( r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, normalizeUnicode( unsafeSQLIdentificatorNaming(column))) filepath = os.path.join( dumpDbPath, "%s-%d.bin" % (_, randomInt(8))) warnMsg = "writing binary ('%s') content to file '%s' " % ( mimetype, filepath) logger.warn(warnMsg) with open(filepath, "wb") as f: _ = safechardecode(value, True) f.write(_) except magic.MagicException as ex: logger.debug(getSafeExString(ex)) if conf.dumpFormat == DUMP_FORMAT.CSV: if field == fields: dataToDumpFile(dumpFP, "%s" % safeCSValue(value)) else: dataToDumpFile( dumpFP, "%s%s" % (safeCSValue(value), conf.csvDel)) elif conf.dumpFormat == DUMP_FORMAT.HTML: dataToDumpFile( dumpFP, "<td>%s</td>" % getUnicode( cgi.escape(value).encode( "ascii", "xmlcharrefreplace"))) field += 1 if conf.dumpFormat == DUMP_FORMAT.SQLITE: try: rtable.insert(values) except SqlmapValueException: pass elif conf.dumpFormat == DUMP_FORMAT.CSV: dataToDumpFile(dumpFP, "\n") elif conf.dumpFormat == DUMP_FORMAT.HTML: dataToDumpFile(dumpFP, "</tr>\n") self._write("|", console=console) self._write("%s\n" % separator) if conf.dumpFormat == DUMP_FORMAT.SQLITE: rtable.endTransaction() logger.info("table '%s.%s' dumped to sqlite3 database '%s'" % (db, table, replication.dbpath)) elif conf.dumpFormat in (DUMP_FORMAT.CSV, DUMP_FORMAT.HTML): if conf.dumpFormat == DUMP_FORMAT.HTML: dataToDumpFile(dumpFP, "</tbody>\n</table>\n</body>\n</html>") else: dataToDumpFile(dumpFP, "\n") dumpFP.close() msg = "table '%s.%s' dumped to %s file '%s'" % ( db, table, conf.dumpFormat, dumpFileName) if not warnFile: logger.info(msg) else: logger.warn(msg)