def retrieve(self, key, unserialize=False):
retVal = None
if key and (self._write_cache or os.path.isfile(self.filepath)):
hash_ = HashDB.hashKey(key)
retVal = self._write_cache.get(hash_)
if not retVal:
for _ in xrange(HASHDB_RETRIEVE_RETRIES):
try:
for row in self.cursor.execute("SELECT value FROM storage WHERE id=?", (hash_,)):
retVal = row[0]
except sqlite3.OperationalError, ex:
if any(_ in getSafeExString(ex) for _ in ("locked", "no such table")):
warnMsg = "problem occurred while accessing session file '%s' ('%s')" % (self.filepath, getSafeExString(ex))
singleTimeWarnMessage(warnMsg)
elif "Could not decode" in getSafeExString(ex):
break
else:
raise
except sqlite3.DatabaseError, ex:
errMsg = "error occurred while accessing session file '%s' ('%s'). " % (self.filepath, getSafeExString(ex))
errMsg += "If the problem persists please rerun with `--flush-session`"
raise SqlmapConnectionException, errMsg
else:
break
time.sleep(1)
def execute(self, query):
retVal = False
try:
self.cursor.execute(query)
retVal = True
except (psycopg2.OperationalError, psycopg2.ProgrammingError) as ex:
logger.warn(("(remote) '%s'" % getSafeExString(ex)).strip())
except psycopg2.InternalError as ex:
raise SqlmapConnectionException(getSafeExString(ex))
self.connector.commit()
return retVal
def adjust(self):
self.closeFP()
if self.index > len(self.filenames):
raise StopIteration
elif self.index == len(self.filenames):
self.iter = iter(self.custom)
else:
self.current = self.filenames[self.index]
if os.path.splitext(self.current)[1].lower() == ".zip":
try:
_ = zipfile.ZipFile(self.current, 'r')
except zipfile.error as ex:
errMsg = "something appears to be wrong with "
errMsg += "the file '%s' ('%s'). Please make " % (self.current, getSafeExString(ex))
errMsg += "sure that you haven't made any changes to it"
raise SqlmapInstallationException(errMsg)
if len(_.namelist()) == 0:
errMsg = "no file(s) inside '%s'" % self.current
raise SqlmapDataException(errMsg)
self.fp = _.open(_.namelist()[0])
else:
self.fp = open(self.current, 'r')
self.iter = iter(self.fp)
self.index += 1
def setOutputFile(self):
self._outputFile = os.path.join(conf.outputPath, "log")
try:
self._outputFP = openFile(self._outputFile, "ab" if not conf.flushSession else "wb")
except IOError, ex:
errMsg = "error occurred while opening log file ('%s')" % getSafeExString(ex)
raise SqlmapGenericException(errMsg)
def crawlThread():
threadData = getCurrentThreadData()
while kb.threadContinue:
with kb.locks.limit:
if threadData.shared.unprocessed:
current = threadData.shared.unprocessed.pop()
if current in visited:
continue
elif conf.crawlExclude and re.search(conf.crawlExclude, current):
dbgMsg = "skipping '%s'" % current
logger.debug(dbgMsg)
continue
else:
visited.add(current)
else:
break
content = None
try:
if current:
content = Request.getPage(url=current, crawling=True, raise404=False)[0]
except SqlmapConnectionException, ex:
errMsg = "connection exception detected ('%s'). skipping " % getSafeExString(ex)
errMsg += "URL '%s'" % current
logger.critical(errMsg)
def connect(self):
def create_sock():
sock = socket.create_connection((self.host, self.port), self.timeout)
if getattr(self, "_tunnel_host", None):
self.sock = sock
self._tunnel()
return sock
success = False
# Reference(s): https://docs.python.org/2/library/ssl.html#ssl.SSLContext
# https://www.mnot.net/blog/2014/12/27/python_2_and_tls_sni
if re.search(r"\A[\d.]+\Z", self.host) is None and kb.tlsSNI.get(self.host) != False and hasattr(ssl, "SSLContext"):
for protocol in filter(lambda _: _ >= ssl.PROTOCOL_TLSv1, _protocols):
try:
sock = create_sock()
context = ssl.SSLContext(protocol)
_ = context.wrap_socket(sock, do_handshake_on_connect=True, server_hostname=self.host)
if _:
success = True
self.sock = _
_protocols.remove(protocol)
_protocols.insert(0, protocol)
break
else:
sock.close()
except (ssl.SSLError, socket.error, httplib.BadStatusLine), ex:
self._tunnel_host = None
logger.debug("SSL connection error occurred ('%s')" % getSafeExString(ex))
if kb.tlsSNI.get(self.host) is None:
kb.tlsSNI[self.host] = success
def server(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, server_name="wsgiref"):
"""
REST-JSON API server
"""
DataStore.admin_id = hexencode(os.urandom(16))
Database.filepath = tempfile.mkstemp(prefix="sqlmapipc-", text=False)[1]
logger.info("Running REST-JSON API server at '%s:%d'.." % (host, port))
logger.info("Admin ID: %s" % DataStore.admin_id)
logger.debug("IPC database: %s" % Database.filepath)
# Initialize IPC database
DataStore.current_db = Database()
DataStore.current_db.connect()
DataStore.current_db.init()
# Run RESTful API
try:
if server_name == "gevent":
from gevent import monkey
monkey.patch_all()
elif server_name == "eventlet":
import eventlet
eventlet.monkey_patch()
logger.debug("use {0} adapter run bottle".format(server_name))
run(host=host, port=port, quiet=True, debug=False, server=server_name)
except socket.error, ex:
if "already in use" in getSafeExString(ex):
logger.error("Address already in use ('%s:%s')" % (host, port))
else:
raise
def connect(self):
self.initConnection()
self.checkFileDb()
try:
self.connector = self.__sqlite.connect(database=self.db, check_same_thread=False, timeout=conf.timeout)
cursor = self.connector.cursor()
cursor.execute("SELECT * FROM sqlite_master")
cursor.close()
except (self.__sqlite.DatabaseError, self.__sqlite.OperationalError):
warnMsg = "unable to connect using SQLite 3 library, trying with SQLite 2"
logger.warn(warnMsg)
try:
try:
import sqlite
except ImportError:
errMsg = "sqlmap requires 'python-sqlite' third-party library "
errMsg += "in order to directly connect to the database '%s'" % self.db
raise SqlmapMissingDependence(errMsg)
self.__sqlite = sqlite
self.connector = self.__sqlite.connect(database=self.db, check_same_thread=False, timeout=conf.timeout)
except (self.__sqlite.DatabaseError, self.__sqlite.OperationalError) as ex:
raise SqlmapConnectionException(getSafeExString(ex))
self.initCursor()
self.printConnected()
def server(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, adapter=RESTAPI_DEFAULT_ADAPTER):
"""
REST-JSON API server
"""
DataStore.admin_id = hexencode(os.urandom(16))
Database.filepath = tempfile.mkstemp(prefix="sqlmapipc-", text=False)[1]
#make adminid to known this is safe because api only avalible to local
file_object = open('/www/xseclab.com/termite/.sqlmapadminid', 'w')
file_object.write(DataStore.admin_id)
file_object.close( )
logger.info("Running REST-JSON API server at '%s:%d'.." % (host, port))
logger.info("Admin ID: %s" % DataStore.admin_id)
logger.debug("IPC database: %s" % Database.filepath)
# Initialize IPC database
DataStore.current_db = Database()
DataStore.current_db.connect()
DataStore.current_db.init()
# Run RESTful API
try:
if adapter == "gevent":
from gevent import monkey
monkey.patch_all()
elif adapter == "eventlet":
import eventlet
eventlet.monkey_patch()
logger.debug("Using adapter '%s' to run bottle" % adapter)
run(host=host, port=port, quiet=True, debug=False, server=adapter)
except socket.error, ex:
if "already in use" in getSafeExString(ex):
logger.error("Address already in use ('%s:%s')" % (host, port))
else:
raise
def connect(self):
self.initConnection()
try:
self.connector = pymysql.connect(host=self.hostname, user=self.user, passwd=self.password, db=self.db, port=self.port, connect_timeout=conf.timeout, use_unicode=True)
except (pymysql.OperationalError, pymysql.InternalError, pymysql.ProgrammingError, struct.error), msg:
raise SqlmapConnectionException(getSafeExString(msg))
def connect(self):
def create_sock():
sock = socket.create_connection((self.host, self.port), self.timeout)
if getattr(self, "_tunnel_host", None):
self.sock = sock
self._tunnel()
return sock
success = False
if not kb.tlsSNI:
for protocol in _protocols:
try:
sock = create_sock()
_ = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=protocol)
if _:
success = True
self.sock = _
_protocols.remove(protocol)
_protocols.insert(0, protocol)
break
else:
sock.close()
except (ssl.SSLError, socket.error, httplib.BadStatusLine), ex:
self._tunnel_host = None
logger.debug("SSL connection error occurred ('%s')" % getSafeExString(ex))
def getConnection(self, host, timeout=None):
try:
# Reference: https://docs.python.org/2/library/ssl.html#ssl.SSLContext.load_cert_chain
return httplib.HTTPSConnection(host, cert_file=self.auth_file, key_file=self.auth_file, timeout=conf.timeout)
except IOError, ex:
errMsg = "error occurred while using key "
errMsg += "file '%s' ('%s')" % (self.auth_file, getSafeExString(ex))
raise SqlmapConnectionException(errMsg)
def loadBoundaries():
try:
doc = et.parse(paths.BOUNDARIES_XML)
except Exception, ex:
errMsg = "something seems to be wrong with "
errMsg += "the file '%s' ('%s'). Please make " % (paths.BOUNDARIES_XML, getSafeExString(ex))
errMsg += "sure that you haven't made any changes to it"
raise SqlmapInstallationException, errMsg
def connect(self):
self.initConnection()
try:
database = "DATABASE=%s;HOSTNAME=%s;PORT=%s;PROTOCOL=TCPIP;" % (self.db, self.hostname, self.port)
self.connector = ibm_db_dbi.connect(database, self.user, self.password)
except ibm_db_dbi.OperationalError, msg:
raise SqlmapConnectionException(getSafeExString(msg))
def execute(self, query):
try:
self.cursor.execute(query)
except (ibm_db_dbi.OperationalError, ibm_db_dbi.ProgrammingError) as ex:
logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex))
except ibm_db_dbi.InternalError as ex:
raise SqlmapConnectionException(getSafeExString(ex))
self.connector.commit()
def execute(self, query):
try:
self.cursor.execute(utf8encode(query))
except self.__sqlite.OperationalError as ex:
logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex))
except self.__sqlite.DatabaseError as ex:
raise SqlmapConnectionException(getSafeExString(ex))
self.connector.commit()
def execute(self, query):
try:
self.cursor.execute(query)
except kinterbasdb.OperationalError as ex:
logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex))
except kinterbasdb.Error as ex:
raise SqlmapConnectionException(getSafeExString(ex))
self.connector.commit()
def __init__(self, dbpath):
try:
self.dbpath = dbpath
self.connection = sqlite3.connect(dbpath)
self.connection.isolation_level = None
self.cursor = self.connection.cursor()
except sqlite3.OperationalError as ex:
errMsg = "error occurred while opening a replication "
errMsg += "file '%s' ('%s')" % (self.filepath, getSafeExString(ex))
raise SqlmapConnectionException(errMsg)
def update():
if not conf.updateAll:
return
success = False
if not os.path.exists(os.path.join(paths.SQLMAP_ROOT_PATH, ".git")):
warnMsg = "not a git repository. It is recommended to clone the 'sqlmapproject/sqlmap' repository "
warnMsg += "from GitHub (e.g. 'git clone --depth 1 %s sqlmap')" % GIT_REPOSITORY
logger.warn(warnMsg)
message = "do you want to try to fetch the latest 'zipball' from repository and extract it (experimental) ? [y/N]"
if readInput(message, default='N', boolean=True):
directory = os.path.abspath(paths.SQLMAP_ROOT_PATH)
try:
open(os.path.join(directory, "sqlmap.py"), "w+b")
except Exception, ex:
errMsg = "unable to update content of directory '%s' ('%s')" % (directory, getSafeExString(ex))
logger.error(errMsg)
else:
for wildcard in ('*', ".*"):
for _ in glob.glob(os.path.join(directory, wildcard)):
try:
if os.path.isdir(_):
shutil.rmtree(_)
else:
os.remove(_)
except:
pass
if glob.glob(os.path.join(directory, '*')):
errMsg = "unable to clear the content of directory '%s'" % directory
logger.error(errMsg)
else:
try:
archive = urllib.urlretrieve(ZIPBALL_PAGE)[0]
with zipfile.ZipFile(archive) as f:
for info in f.infolist():
info.filename = re.sub(r"\Asqlmap[^/]+", "", info.filename)
if info.filename:
f.extract(info, directory)
filepath = os.path.join(paths.SQLMAP_ROOT_PATH, "lib", "core", "settings.py")
if os.path.isfile(filepath):
with open(filepath, "rb") as f:
version = re.search(r"(?m)^VERSION\s*=\s*['\"]([^'\"]+)", f.read()).group(1)
logger.info("updated to the latest version '%s#dev'" % version)
success = True
except Exception, ex:
logger.error("update could not be completed ('%s')" % getSafeExString(ex))
else:
if not success:
logger.error("update could not be completed")
def dbTableValues(self, tableValues):
replication = None
rtable = None
dumpFP = None
appendToFile = False
warnFile = False
if tableValues is None:
return
db = tableValues["__infos__"]["db"]
if not db:
db = "All"
table = tableValues["__infos__"]["table"]
if hasattr(conf, "api"):
self._write(tableValues, content_type=CONTENT_TYPE.DUMP_TABLE)
return
dumpDbPath = os.path.join(conf.dumpPath, unsafeSQLIdentificatorNaming(db))
if conf.dumpFormat == DUMP_FORMAT.SQLITE:
replication = Replication(os.path.join(conf.dumpPath, "%s.sqlite3" % unsafeSQLIdentificatorNaming(db)))
elif conf.dumpFormat in (DUMP_FORMAT.CSV, DUMP_FORMAT.HTML):
if not os.path.isdir(dumpDbPath):
try:
os.makedirs(dumpDbPath, 0755)
except:
warnFile = True
_ = unicodeencode(re.sub(r"[^\w]", "_", unsafeSQLIdentificatorNaming(db)))
dumpDbPath = os.path.join(
conf.dumpPath, "%s-%s" % (_, hashlib.md5(unicodeencode(db)).hexdigest()[:8])
)
if not os.path.isdir(dumpDbPath):
try:
os.makedirs(dumpDbPath, 0755)
except Exception, ex:
try:
tempDir = tempfile.mkdtemp(prefix="sqlmapdb")
except IOError, _:
errMsg = "unable to write to the temporary directory ('%s'). " % _
errMsg += "Please make sure that your disk is not full and "
errMsg += "that you have sufficient write permissions to "
errMsg += "create temporary files and/or directories"
raise SqlmapSystemException(errMsg)
warnMsg = "unable to create dump directory "
warnMsg += "'%s' (%s). " % (dumpDbPath, getSafeExString(ex))
warnMsg += "Using temporary directory '%s' instead" % tempDir
logger.warn(warnMsg)
dumpDbPath = tempDir
def connect(self):
self.initConnection()
try:
self.connector = psycopg2.connect(host=self.hostname, user=self.user, password=self.password, database=self.db, port=self.port)
except psycopg2.OperationalError as ex:
raise SqlmapConnectionException(getSafeExString(ex))
self.connector.set_client_encoding('UNICODE')
self.initCursor()
self.printConnected()
def execute(self, query):
retVal = False
try:
self.cursor.execute(utf8encode(query))
retVal = True
except (pymssql.OperationalError, pymssql.ProgrammingError) as ex:
logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex).replace("\n", " "))
except pymssql.InternalError as ex:
raise SqlmapConnectionException(getSafeExString(ex))
return retVal
def connect(self):
self.initConnection()
try:
msg = "what's the location of 'hsqldb.jar'? "
jar = readInput(msg)
checkFile(jar)
args = "-Djava.class.path=%s" % jar
jvm_path = jpype.getDefaultJVMPath()
jpype.startJVM(jvm_path, args)
except Exception as ex:
raise SqlmapConnectionException(getSafeExString(ex))
try:
driver = 'org.hsqldb.jdbc.JDBCDriver'
connection_string = 'jdbc:hsqldb:mem:.' # 'jdbc:hsqldb:hsql://%s/%s' % (self.hostname, self.db)
self.connector = jaydebeapi.connect(driver, connection_string, str(self.user), str(self.password))
except Exception as ex:
raise SqlmapConnectionException(getSafeExString(ex))
self.initCursor()
self.printConnected()
def execute(self, statement, arguments=None):
while True:
try:
if arguments:
self.cursor.execute(statement, arguments)
else:
self.cursor.execute(statement)
except sqlite3.OperationalError, ex:
if not "locked" in getSafeExString(ex):
raise
else:
break
def connect(self):
self.initConnection()
try:
self.connector = pymssql.connect(host="%s:%d" % (self.hostname, self.port), user=self.user, password=self.password, database=self.db, login_timeout=conf.timeout, timeout=conf.timeout)
except (pymssql.Error, _mssql.MssqlDatabaseException) as ex:
raise SqlmapConnectionException(getSafeExString(ex))
except ValueError:
raise SqlmapConnectionException
self.initCursor()
self.printConnected()
def server(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, adapter=RESTAPI_DEFAULT_ADAPTER, username=None, password=None):
"""
REST-JSON API server
"""
DataStore.admin_token = hexencode(os.urandom(16))
DataStore.username = username
DataStore.password = password
_, Database.filepath = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.IPC, text=False)
os.close(_)
if port == 0: # random
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
s.bind((host, 0))
port = s.getsockname()[1]
logger.info("Running REST-JSON API server at '%s:%d'.." % (host, port))
logger.info("Admin (secret) token: %s" % DataStore.admin_token)
logger.debug("IPC database: '%s'" % Database.filepath)
# Initialize IPC database
DataStore.current_db = Database()
DataStore.current_db.connect()
DataStore.current_db.init()
# Run RESTful API
try:
# Supported adapters: aiohttp, auto, bjoern, cgi, cherrypy, diesel, eventlet, fapws3, flup, gae, gevent, geventSocketIO, gunicorn, meinheld, paste, rocket, tornado, twisted, waitress, wsgiref
# Reference: https://bottlepy.org/docs/dev/deployment.html || bottle.server_names
if adapter == "gevent":
from gevent import monkey
monkey.patch_all()
elif adapter == "eventlet":
import eventlet
eventlet.monkey_patch()
logger.debug("Using adapter '%s' to run bottle" % adapter)
run(host=host, port=port, quiet=True, debug=True, server=adapter)
except socket.error as ex:
if "already in use" in getSafeExString(ex):
logger.error("Address already in use ('%s:%s')" % (host, port))
else:
raise
except ImportError:
if adapter.lower() not in server_names:
errMsg = "Adapter '%s' is unknown. " % adapter
errMsg += "List of supported adapters: %s" % ', '.join(sorted(list(server_names.keys())))
else:
errMsg = "Server support for adapter '%s' is not installed on this system " % adapter
errMsg += "(Note: you can try to install it with 'sudo apt-get install python-%s' or 'sudo pip install %s')" % (adapter, adapter)
logger.critical(errMsg)
def _get_cursor(self):
threadData = getCurrentThreadData()
if threadData.hashDBCursor is None:
try:
connection = sqlite3.connect(self.filepath, timeout=3, isolation_level=None)
threadData.hashDBCursor = connection.cursor()
threadData.hashDBCursor.execute("CREATE TABLE IF NOT EXISTS storage (id INTEGER PRIMARY KEY, value TEXT)")
connection.commit()
except Exception, ex:
errMsg = "error occurred while opening a session "
errMsg += "file '%s' ('%s')" % (self.filepath, getSafeExString(ex))
raise SqlmapDataException(errMsg)
def connect(self):
self.initConnection()
if not self.hostname:
self.checkFileDb()
try:
# Reference: http://www.daniweb.com/forums/thread248499.html
self.connector = kinterbasdb.connect(host=self.hostname.encode(UNICODE_ENCODING), database=self.db.encode(UNICODE_ENCODING), user=self.user.encode(UNICODE_ENCODING), password=self.password.encode(UNICODE_ENCODING), charset="UTF8")
except kinterbasdb.OperationalError as ex:
raise SqlmapConnectionException(getSafeExString(ex))
self.initCursor()
self.printConnected()
def next(self):
retVal = None
while True:
self.counter += 1
try:
retVal = self.iter.next().rstrip()
except zipfile.error, ex:
errMsg = "something seems to be wrong with "
errMsg += "the file '%s' ('%s'). Please make " % (self.current, getSafeExString(ex))
errMsg += "sure that you haven't made any changes to it"
raise SqlmapInstallationException, errMsg
except StopIteration:
self.adjust()
retVal = self.iter.next().rstrip()
def loadPayloads():
for payloadFile in PAYLOAD_XML_FILES:
payloadFilePath = os.path.join(paths.SQLMAP_XML_PAYLOADS_PATH, payloadFile)
try:
doc = et.parse(payloadFilePath)
except Exception, ex:
errMsg = "something appears to be wrong with "
errMsg += "the file '%s' ('%s'). Please make " % (payloadFilePath, getSafeExString(ex))
errMsg += "sure that you haven't made any changes to it"
raise SqlmapInstallationException(errMsg)
root = doc.getroot()
parseXmlNode(root)
def _saveToResultsFile():
if not conf.resultsFP:
return
results = {}
techniques = dict((_[1], _[0]) for _ in getPublicTypeMembers(PAYLOAD.TECHNIQUE))
for injection in kb.injections + kb.falsePositives:
if injection.place is None or injection.parameter is None:
continue
key = (injection.place, injection.parameter, ';'.join(injection.notes))
if key not in results:
results[key] = []
results[key].extend(list(injection.data.keys()))
try:
for key, value in results.items():
place, parameter, notes = key
line = "%s,%s,%s,%s,%s%s" % (safeCSValue(kb.originalUrls.get(conf.url) or conf.url), place, parameter, "".join(techniques[_][0].upper() for _ in sorted(value)), notes, os.linesep)
conf.resultsFP.write(line)
if not results:
line = "%s,,,,%s" % (conf.url, os.linesep)
conf.resultsFP.write(line)
conf.resultsFP.flush()
except IOError as ex:
errMsg = "unable to write to the results file '%s' ('%s'). " % (conf.resultsFilename, getSafeExString(ex))
raise SqlmapSystemException(errMsg)