def storeHashesToFile(attack_dict): if not attack_dict: return items = OrderedSet() for user, hashes in attack_dict.items(): for hash_ in hashes: hash_ = hash_.split()[0] if hash_ and hash_.strip() else hash_ if hash_ and hash_ != NULL and hashRecognition(hash_): item = None if user and not user.startswith(DUMMY_USER_PREFIX): item = "%s:%s\n" % (user.encode(UNICODE_ENCODING), hash_.encode(UNICODE_ENCODING)) else: item = "%s\n" % hash_.encode(UNICODE_ENCODING) if item and item not in items: items.add(item) if kb.storeHashesChoice is None: message = "do you want to store hashes to a temporary file " message += "for eventual further processing with other tools [y/N] " kb.storeHashesChoice = readInput(message, default='N', boolean=True) if items and kb.storeHashesChoice: handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.HASHES, suffix=".txt") os.close(handle) infoMsg = "writing hashes to a temporary file '%s' " % filename logger.info(infoMsg) with open(filename, "w+") as f: for item in items: f.write(item)
def tamper(payload, **kwargs): """ Adds multiple spaces (' ') around SQL keywords Notes: * Useful to bypass very weak and bespoke web application firewalls that has poorly written permissive regular expressions Reference: https://www.owasp.org/images/7/74/Advanced_SQL_Injection.ppt >>> random.seed(0) >>> tamper('1 UNION SELECT foobar') '1 UNION SELECT foobar' """ retVal = payload if payload: words = OrderedSet() for match in re.finditer(r"\b[A-Za-z_]+\b", payload): word = match.group() if word.upper() in kb.keywords: words.add(word) for word in words: retVal = re.sub(r"(?<=\W)%s(?=[^A-Za-z_(]|\Z)" % word, "%s%s%s" % (' ' * random.randint(1, 4), word, ' ' * random.randint(1, 4)), retVal) retVal = re.sub(r"(?<=\W)%s(?=[(])" % word, "%s%s" % (' ' * random.randint(1, 4), word), retVal) return retVal
def parseSitemap(url, retVal=None): global abortedFlag if retVal is not None: logger.debug("parsing sitemap '%s'" % url) try: if retVal is None: abortedFlag = False retVal = OrderedSet() try: content = Request.getPage( url=url, raise404=True)[0] if not abortedFlag else "" except _http_client.InvalidURL: errMsg = "invalid URL given for sitemap ('%s')" % url raise SqlmapSyntaxException(errMsg) for match in re.finditer(r"<loc>\s*([^<]+)", content or ""): if abortedFlag: break url = match.group(1).strip() if url.endswith(".xml") and "sitemap" in url.lower(): if kb.followSitemapRecursion is None: message = "sitemap recursion detected. Do you want to follow? [y/N] " kb.followSitemapRecursion = readInput(message, default='N', boolean=True) if kb.followSitemapRecursion: parseSitemap(url, retVal) else: retVal.add(url) except KeyboardInterrupt: abortedFlag = True warnMsg = "user aborted during sitemap parsing. sqlmap " warnMsg += "will use partial list" logger.warn(warnMsg) return retVal
def parseSitemap(url, retVal=None): global abortedFlag if retVal is not None: logger.debug("parsing sitemap '%s'" % url) try: if retVal is None: abortedFlag = False retVal = OrderedSet() try: content = Request.getPage(url=url, raise404=True)[0] if not abortedFlag else "" except _http_client.InvalidURL: errMsg = "invalid URL given for sitemap ('%s')" % url raise SqlmapSyntaxException(errMsg) for match in re.finditer(r"<loc>\s*([^<]+)", content or ""): if abortedFlag: break url = match.group(1).strip() if url.endswith(".xml") and "sitemap" in url.lower(): if kb.followSitemapRecursion is None: message = "sitemap recursion detected. Do you want to follow? [y/N] " kb.followSitemapRecursion = readInput(message, default='N', boolean=True) if kb.followSitemapRecursion: parseSitemap(url, retVal) else: retVal.add(url) except KeyboardInterrupt: abortedFlag = True warnMsg = "user aborted during sitemap parsing. sqlmap " warnMsg += "will use partial list" logger.warn(warnMsg) return retVal
def crawl(target): try: visited = set() threadData = getCurrentThreadData() threadData.shared.value = OrderedSet() threadData.shared.formsFound = False def crawlThread(): threadData = getCurrentThreadData() while kb.threadContinue: with kb.locks.limit: if threadData.shared.unprocessed: current = threadData.shared.unprocessed.pop() if current in visited: continue elif conf.crawlExclude and re.search(conf.crawlExclude, current): dbgMsg = "skipping '%s'" % current logger.debug(dbgMsg) continue else: visited.add(current) else: break content = None try: if current: content = Request.getPage(url=current, crawling=True, raise404=False)[0] except SqlmapConnectionException as ex: errMsg = "connection exception detected ('%s'). skipping " % getSafeExString(ex) errMsg += "URL '%s'" % current logger.critical(errMsg) except SqlmapSyntaxException: errMsg = "invalid URL detected. skipping '%s'" % current logger.critical(errMsg) except _http_client.InvalidURL as ex: errMsg = "invalid URL detected ('%s'). skipping " % getSafeExString(ex) errMsg += "URL '%s'" % current logger.critical(errMsg) if not kb.threadContinue: break if isinstance(content, six.text_type): try: match = re.search(r"(?si)<html[^>]*>(.+)</html>", content) if match: content = "<html>%s</html>" % match.group(1) soup = BeautifulSoup(content) tags = soup('a') tags += re.finditer(r'(?i)\s(href|src)=["\'](?P<href>[^>"\']+)', content) tags += re.finditer(r'(?i)window\.open\(["\'](?P<href>[^)"\']+)["\']', content) for tag in tags: href = tag.get("href") if hasattr(tag, "get") else tag.group("href") if href: if threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID: current = threadData.lastRedirectURL[1] url = _urllib.parse.urljoin(current, htmlUnescape(href)) # flag to know if we are dealing with the same target host _ = checkSameHost(url, target) if conf.scope: if not re.search(conf.scope, url, re.I): continue elif not _: continue if (extractRegexResult(r"\A[^?]+\.(?P<result>\w+)(\?|\Z)", url) or "").lower() not in CRAWL_EXCLUDE_EXTENSIONS: with kb.locks.value: threadData.shared.deeper.add(url) if re.search(r"(.*?)\?(.+)", url) and not re.search(r"\?(v=)?\d+\Z", url) and not re.search(r"(?i)\.(js|css)(\?|\Z)", url): threadData.shared.value.add(url) except UnicodeEncodeError: # for non-HTML files pass except ValueError: # for non-valid links pass finally: if conf.forms: threadData.shared.formsFound |= len(findPageForms(content, current, False, True)) > 0 if conf.verbose in (1, 2): threadData.shared.count += 1 status = '%d/%d links visited (%d%%)' % (threadData.shared.count, threadData.shared.length, round(100.0 * threadData.shared.count / threadData.shared.length)) dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status), True) threadData.shared.deeper = set() threadData.shared.unprocessed = set([target]) if kb.checkSitemap is None: message = "do you want to check for the existence of " message += "site's sitemap(.xml) [y/N] " kb.checkSitemap = readInput(message, default='N', boolean=True) if kb.checkSitemap: found = True items = None url = _urllib.parse.urljoin(target, "/sitemap.xml") try: items = parseSitemap(url) except SqlmapConnectionException as ex: if "page not found" in getSafeExString(ex): found = False logger.warn("'sitemap.xml' not found") except: pass finally: if found: if items: for item in items: if re.search(r"(.*?)\?(.+)", item): threadData.shared.value.add(item) if conf.crawlDepth > 1: threadData.shared.unprocessed.update(items) logger.info("%s links found" % ("no" if not items else len(items))) if not conf.bulkFile: infoMsg = "starting crawler for target URL '%s'" % target logger.info(infoMsg) for i in xrange(conf.crawlDepth): threadData.shared.count = 0 threadData.shared.length = len(threadData.shared.unprocessed) numThreads = min(conf.threads, len(threadData.shared.unprocessed)) if not conf.bulkFile: logger.info("searching for links with depth %d" % (i + 1)) runThreads(numThreads, crawlThread, threadChoice=(i > 0)) clearConsoleLine(True) if threadData.shared.deeper: threadData.shared.unprocessed = set(threadData.shared.deeper) else: break except KeyboardInterrupt: warnMsg = "user aborted during crawling. sqlmap " warnMsg += "will use partial list" logger.warn(warnMsg) finally: clearConsoleLine(True) if not threadData.shared.value: if not (conf.forms and threadData.shared.formsFound): warnMsg = "no usable links found (with GET parameters)" if conf.forms: warnMsg += " or forms" logger.warn(warnMsg) else: for url in threadData.shared.value: kb.targets.add((urldecode(url, kb.pageEncoding), None, None, None, None)) if kb.targets: if kb.normalizeCrawlingChoice is None: message = "do you want to normalize " message += "crawling results [Y/n] " kb.normalizeCrawlingChoice = readInput(message, default='Y', boolean=True) if kb.normalizeCrawlingChoice: seen = set() results = OrderedSet() for target in kb.targets: if target[1] in (HTTPMETHOD.GET, None): match = re.search(r"/[^/?]*\?.*\Z", target[0]) if match: key = re.sub(r"=[^=&]*", "=", match.group(0)).strip('&') if key not in seen: results.add(target) seen.add(key) else: results.add(target) kb.targets = results storeResultsToFile(kb.targets)
def attackDumpedTable(): if kb.data.dumpedTable: table = kb.data.dumpedTable columns = list(table.keys()) count = table["__infos__"]["count"] if not count: return debugMsg = "analyzing table dump for possible password hashes" logger.debug(debugMsg) found = False col_user = '' col_passwords = set() attack_dict = {} binary_fields = OrderedSet() replacements = {} for column in sorted(columns, key=len, reverse=True): if column and column.lower() in COMMON_USER_COLUMNS: col_user = column break for column in columns: if column != "__infos__" and table[column]["values"]: if all(INVALID_UNICODE_CHAR_FORMAT.split('%')[0] in (value or "") for value in table[column]["values"]): binary_fields.add(column) if binary_fields: _ = ','.join(binary_fields) warnMsg = "potential binary fields detected ('%s'). In case of any problems you are " % _ warnMsg += "advised to rerun table dump with '--fresh-queries --binary-fields=\"%s\"'" % _ logger.warn(warnMsg) for i in xrange(count): if not found and i > HASH_RECOGNITION_QUIT_THRESHOLD: break for column in columns: if column == col_user or column == "__infos__": continue if len(table[column]["values"]) <= i: continue if conf.binaryFields and column in conf.binaryFields: continue value = table[column]["values"][i] if column in binary_fields and re.search(HASH_BINARY_COLUMNS_REGEX, column) is not None: previous = value value = encodeHex(getBytes(value), binary=False) replacements[value] = previous if hashRecognition(value): found = True if col_user and i < len(table[col_user]["values"]): if table[col_user]["values"][i] not in attack_dict: attack_dict[table[col_user]["values"][i]] = [] attack_dict[table[col_user]["values"][i]].append(value) else: attack_dict["%s%d" % (DUMMY_USER_PREFIX, i)] = [value] col_passwords.add(column) if attack_dict: infoMsg = "recognized possible password hashes in column%s " % ("s" if len(col_passwords) > 1 else "") infoMsg += "'%s'" % ", ".join(col for col in col_passwords) logger.info(infoMsg) storeHashesToFile(attack_dict) message = "do you want to crack them via a dictionary-based attack? %s" % ("[y/N/q]" if conf.multipleTargets else "[Y/n/q]") choice = readInput(message, default='N' if conf.multipleTargets else 'Y').upper() if choice == 'N': return elif choice == 'Q': raise SqlmapUserQuitException results = dictionaryAttack(attack_dict) lut = dict() for (_, hash_, password) in results: if hash_: key = hash_ if hash_ not in replacements else replacements[hash_] lut[key.lower()] = password lut["0x%s" % key.lower()] = password debugMsg = "post-processing table dump" logger.debug(debugMsg) for i in xrange(count): for column in columns: if not (column == col_user or column == '__infos__' or len(table[column]['values']) <= i): value = table[column]['values'][i] if value and value.lower() in lut: table[column]['values'][i] = "%s (%s)" % (getUnicode(table[column]['values'][i]), getUnicode(lut[value.lower()] or HASH_EMPTY_PASSWORD_MARKER)) table[column]['length'] = max(table[column]['length'], len(table[column]['values'][i]))