def getPage(**kwargs):
"""
This method connects to the target url or proxy and returns
the target url page content
"""
if conf.delay is not None and isinstance(conf.delay, (int, float)) and conf.delay > 0:
time.sleep(conf.delay)
elif conf.cpuThrottle:
delay = 0.00001 * (conf.cpuThrottle ** 2)
time.sleep(delay)
kb.locks.reqLock.acquire()
kb.lastRequestUID += 1
requestID = kb.lastRequestUID
kb.locks.reqLock.release()
url = kwargs.get('url', conf.url).replace(" ", "%20")
get = kwargs.get('get', None)
post = kwargs.get('post', None)
method = kwargs.get('method', None)
cookie = kwargs.get('cookie', None)
ua = kwargs.get('ua', None)
direct = kwargs.get('direct', False)
multipart = kwargs.get('multipart', False)
silent = kwargs.get('silent', False)
raise404 = kwargs.get('raise404', True)
auxHeaders = kwargs.get('auxHeaders', None)
response = kwargs.get('response', False)
page = ""
cookieStr = ""
requestMsg = "HTTP request [#%d]:\n%s " % (requestID, conf.method)
requestMsg += "%s" % urlparse.urlsplit(url)[2] or "/"
responseMsg = "HTTP response "
requestHeaders = ""
responseHeaders = ""
logHeaders = ""
try:
if silent:
socket.setdefaulttimeout(3)
if direct:
if "?" in url:
url, params = url.split("?")
params = urlencode(params)
url = "%s?%s" % (url, params)
requestMsg += "?%s" % params
elif multipart:
# Needed in this form because of potential circle dependency
# problem (option -> update -> connect -> option)
from lib.core.option import proxyHandler
multipartOpener = urllib2.build_opener(proxyHandler, multipartpost.MultipartPostHandler)
conn = multipartOpener.open(url, multipart)
page = conn.read()
responseHeaders = conn.info()
page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type"))
return page
else:
if conf.parameters.has_key(PLACE.GET) and not get:
get = conf.parameters[PLACE.GET]
if get:
url = "%s?%s" % (url, get)
requestMsg += "?%s" % get
if conf.method == HTTPMETHOD.POST:
if conf.parameters.has_key(PLACE.POST) and not post:
post = conf.parameters[PLACE.POST]
requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str
# Perform HTTP request
headers = forgeHeaders(cookie, ua)
if kb.authHeader:
headers["Authorization"] = kb.authHeader
if kb.proxyAuthHeader:
headers["Proxy-authorization"] = kb.proxyAuthHeader
if auxHeaders:
for key, item in auxHeaders.items():
headers[key] = item
if method:
req = MethodRequest(url, post, headers)
req.set_method(method)
else:
req = urllib2.Request(url, post, headers)
if not conf.dropSetCookie and conf.cj:
for _, cookie in enumerate(conf.cj):
if not cookieStr:
cookieStr = "Cookie: "
cookie = getUnicode(cookie)
index = cookie.index(" for ")
cookieStr += "%s; " % cookie[8:index]
conn = urllib2.urlopen(req)
if not req.has_header("Accept-Encoding"):
requestHeaders += "Accept-Encoding: identity\n"
requestHeaders += "\n".join(["%s: %s" % (header, value) for header, value in req.header_items()])
if not req.has_header("Cookie") and cookieStr:
requestHeaders += "\n%s" % cookieStr[:-2]
if not req.has_header("Connection"):
requestHeaders += "\nConnection: close"
requestMsg += "\n%s" % requestHeaders
if post:
requestMsg += "\n%s" % post
requestMsg += "\n"
logger.log(8, requestMsg)
if not kb.authHeader and req.has_header("Authorization"):
kb.authHeader = req.get_header("Authorization")
if not kb.proxyAuthHeader and req.has_header("Proxy-authorization"):
kb.proxyAuthHeader = req.get_header("Proxy-authorization")
if hasattr(conn, "redurl") and hasattr(conn, "redcode") and not conf.redirectHandled:
msg = "sqlmap got a %d redirect to " % conn.redcode
msg += "%s - What target address do you " % conn.redurl
msg += "want to use from now on? %s " % conf.url
msg += "(default) or provide another target address based "
msg += "also on the redirection got from the application\n"
while True:
choice = readInput(msg, default="1")
if not choice or choice == "1":
pass
else:
conf.url = choice
return Connect.__getPageProxy(**kwargs)
break
conf.redirectHandled = True
# Reset the number of connection retries
conf.retriesCount = 0
# Return response object
if response:
return conn, None
# Get HTTP response
page = conn.read()
code = conn.code
status = conn.msg
responseHeaders = conn.info()
page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type"))
except urllib2.HTTPError, e:
code = e.code
status = e.msg
try:
page = e.read()
responseHeaders = e.info()
except socket.timeout:
warnMsg = "connection timed out while trying "
warnMsg += "to get error page information (%d)" % code
logger.warn(warnMsg)
return None, None
except:
pass
responseMsg = "\n%s[#%d] (%d %s):\n" % (responseMsg, requestID, code, status)
if responseHeaders:
logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, value) for (key, value) in responseHeaders.items()])
logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page))
if e.code == 401:
errMsg = "not authorized, try to provide right HTTP "
errMsg += "authentication type and valid credentials (%d)" % code
raise sqlmapConnectionException, errMsg
elif e.code == 404 and raise404:
errMsg = "page not found (%d)" % code
raise sqlmapConnectionException, errMsg
else:
debugMsg = "got HTTP error code: %d (%s)" % (code, status)
logger.debug(debugMsg)
def getPage(**kwargs):
"""
This method connects to the target url or proxy and returns
the target url page content
"""
if conf.delay is not None and isinstance(conf.delay, (int, float)) and conf.delay > 0:
time.sleep(conf.delay)
elif conf.cpuThrottle:
cpuThrottle(conf.cpuThrottle)
threadData = getCurrentThreadData()
threadData.lastRequestUID += 1
# dirty hack because urllib2 just skips the other part of provided url
# splitted with space char while urlencoding it in the later phase
url = kwargs.get('url', conf.url).replace(" ", "%20")
get = kwargs.get('get', None)
post = kwargs.get('post', None)
method = kwargs.get('method', None)
cookie = kwargs.get('cookie', None)
ua = kwargs.get('ua', None)
referer = kwargs.get('referer', None)
direct = kwargs.get('direct', False)
multipart = kwargs.get('multipart', False)
silent = kwargs.get('silent', False)
raise404 = kwargs.get('raise404', True)
auxHeaders = kwargs.get('auxHeaders', None)
response = kwargs.get('response', False)
ignoreTimeout = kwargs.get('ignoreTimeout', False)
refreshing = kwargs.get('refreshing', False)
page = ""
cookieStr = ""
requestMsg = "HTTP request [#%d]:\n%s " % (threadData.lastRequestUID, conf.method)
requestMsg += "%s" % urlparse.urlsplit(url)[2] or "/"
responseMsg = "HTTP response "
requestHeaders = ""
responseHeaders = None
logHeaders = ""
try:
if silent:
socket.setdefaulttimeout(HTTP_SILENT_TIMEOUT)
if direct:
if "?" in url:
url, params = url.split("?")
params = urlencode(params)
url = "%s?%s" % (url, params)
requestMsg += "?%s" % params
elif multipart:
# Needed in this form because of potential circle dependency
# problem (option -> update -> connect -> option)
from lib.core.option import proxyHandler
multipartOpener = urllib2.build_opener(proxyHandler, multipartpost.MultipartPostHandler)
conn = multipartOpener.open(url, multipart)
page = conn.read()
responseHeaders = conn.info()
responseHeaders[URI_HTTP_HEADER] = conn.geturl()
page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
return page
elif refreshing:
# Reference(s):
# http://vancouver-webpages.com/META/metatags.detail.html
# http://webdesign.about.com/od/metataglibraries/a/aa080300a.htm
get = None
post = None
else:
if conf.parameters.has_key(PLACE.GET) and not get:
get = conf.parameters[PLACE.GET]
if get:
url = "%s?%s" % (url, get)
requestMsg += "?%s" % get
if conf.method == HTTPMETHOD.POST:
if conf.parameters.has_key(PLACE.POST) and not post:
post = conf.parameters[PLACE.POST]
requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str
# Perform HTTP request
headers = forgeHeaders(cookie, ua, referer)
if conf.realTest:
headers[HTTPHEADER.REFERER] = "%s://%s" % (conf.scheme, conf.hostname)
if kb.authHeader:
headers[HTTPHEADER.AUTHORIZATION] = kb.authHeader
if kb.proxyAuthHeader:
headers[HTTPHEADER.PROXY_AUTHORIZATION] = kb.proxyAuthHeader
if auxHeaders:
for key, item in auxHeaders.items():
headers[key] = item
for key, item in headers.items():
del headers[key]
headers[encodeUnicode(key, kb.pageEncoding)] = encodeUnicode(item, kb.pageEncoding)
post = encodeUnicode(post, kb.pageEncoding)
if method:
req = MethodRequest(url, post, headers)
req.set_method(method)
else:
req = urllib2.Request(url, post, headers)
if not conf.dropSetCookie and conf.cj:
for _, cookie in enumerate(conf.cj):
if not cookieStr:
cookieStr = "Cookie: "
cookie = getUnicode(cookie)
index = cookie.index(" for ")
cookieStr += "%s; " % cookie[8:index]
if not req.has_header(HTTPHEADER.ACCEPT_ENCODING):
requestHeaders += "%s: identity\n" % HTTPHEADER.ACCEPT_ENCODING
requestHeaders += "\n".join(["%s: %s" % (header, value) for header, value in req.header_items()])
if not req.has_header(HTTPHEADER.COOKIE) and cookieStr:
requestHeaders += "\n%s" % cookieStr[:-2]
if not req.has_header(HTTPHEADER.CONNECTION):
requestHeaders += "\n%s: close" % HTTPHEADER.CONNECTION
requestMsg += "\n%s" % requestHeaders
if post:
requestMsg += "\n\n%s" % post
requestMsg += "\n"
logger.log(8, requestMsg)
conn = urllib2.urlopen(req)
if not kb.authHeader and req.has_header(HTTPHEADER.AUTHORIZATION):
kb.authHeader = req.get_header(HTTPHEADER.AUTHORIZATION)
if not kb.proxyAuthHeader and req.has_header(HTTPHEADER.PROXY_AUTHORIZATION):
kb.proxyAuthHeader = req.get_header(HTTPHEADER.PROXY_AUTHORIZATION)
if hasattr(conn, "setcookie"):
kb.redirectSetCookie = conn.setcookie
if hasattr(conn, "redurl") and hasattr(conn, "redcode") and not conf.redirectHandled and not conf.realTest:
msg = "sqlmap got a %d redirect to " % conn.redcode
msg += "%s - What target address do you " % conn.redurl
msg += "want to use from now on? %s " % conf.url
msg += "(default) or provide another target address based "
msg += "also on the redirection got from the application\n"
while True:
choice = readInput(msg, default=None)
if not choice:
pass
else:
conf.url = choice
try:
parseTargetUrl()
return Connect.__getPageProxy(**kwargs)
except sqlmapSyntaxException:
continue
break
conf.redirectHandled = True
# Reset the number of connection retries
kb.retriesCount = 0
# Return response object
if response:
return conn, None
# Get HTTP response
page = conn.read()
code = conn.code
responseHeaders = conn.info()
responseHeaders[URI_HTTP_HEADER] = conn.geturl()
page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
status = getUnicode(conn.msg)
if extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE) and not refreshing:
url = extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE)
if url.lower().startswith('http://'):
kwargs['url'] = url
else:
kwargs['url'] = conf.url[:conf.url.rfind('/')+1] + url
threadData.lastRedirectMsg = (threadData.lastRequestUID, page)
kwargs['refreshing'] = True
debugMsg = "got HTML meta refresh header"
logger.debug(debugMsg)
try:
return Connect.__getPageProxy(**kwargs)
except sqlmapSyntaxException:
pass
# Explicit closing of connection object
if not conf.keepAlive:
try:
conn.fp._sock.close()
conn.close()
except Exception, msg:
warnMsg = "problem occured during connection closing ('%s')" % msg
logger.warn(warnMsg)
except urllib2.HTTPError, e:
page = None
responseHeaders = None
try:
page = e.read()
responseHeaders = e.info()
responseHeaders[URI_HTTP_HEADER] = e.geturl()
page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
except socket.timeout:
warnMsg = "connection timed out while trying "
warnMsg += "to get error page information (%d)" % e.code
logger.warn(warnMsg)
return None, None
except:
pass
code = e.code
threadData.lastHTTPError = (threadData.lastRequestUID, code)
if code not in kb.httpErrorCodes:
kb.httpErrorCodes[code] = 0
kb.httpErrorCodes[code] += 1
status = getUnicode(e.msg)
responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status)
if responseHeaders:
logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in responseHeaders.items()])
logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page if isinstance(page, unicode) else getUnicode(page)))
if conf.verbose <= 5:
responseMsg += getUnicode(logHeaders)
elif conf.verbose > 5:
responseMsg += "%s\n\n%s\n" % (logHeaders, page)
logger.log(7, responseMsg)
if e.code == 401:
errMsg = "not authorized, try to provide right HTTP "
errMsg += "authentication type and valid credentials (%d)" % code
raise sqlmapConnectionException, errMsg
elif e.code == 404 and raise404:
errMsg = "page not found (%d)" % code
raise sqlmapConnectionException, errMsg
else:
debugMsg = "got HTTP error code: %d (%s)" % (code, status)
logger.debug(debugMsg)
page = processResponse(page, responseHeaders)
return page, responseHeaders
def getPage(**kwargs):
"""
This method connects to the target url or proxy and returns
the target url page content
"""
if conf.delay is not None and isinstance(conf.delay, (int, float)) and conf.delay > 0:
time.sleep(conf.delay)
elif conf.cpuThrottle:
cpuThrottle(conf.cpuThrottle)
threadData = getCurrentThreadData()
threadData.lastRequestUID += 1
url = kwargs.get('url', conf.url)
get = kwargs.get('get', None)
post = kwargs.get('post', None)
method = kwargs.get('method', None)
cookie = kwargs.get('cookie', None)
ua = kwargs.get('ua', None)
referer = kwargs.get('referer', None)
host = kwargs.get('host', conf.host)
direct = kwargs.get('direct', False)
multipart = kwargs.get('multipart', False)
silent = kwargs.get('silent', False)
raise404 = kwargs.get('raise404', True)
auxHeaders = kwargs.get('auxHeaders', None)
response = kwargs.get('response', False)
ignoreTimeout = kwargs.get('ignoreTimeout', kb.ignoreTimeout)
refreshing = kwargs.get('refreshing', False)
retrying = kwargs.get('retrying', False)
crawling = kwargs.get('crawling', False)
if not urlparse.urlsplit(url).netloc:
url = urlparse.urljoin(conf.url, url)
# flag to know if we are dealing with the same target host
target = reduce(lambda x, y: x == y, map(lambda x: urlparse.urlparse(x).netloc.split(':')[0], [url, conf.url or ""]))
if not retrying:
# Reset the number of connection retries
threadData.retriesCount = 0
# fix for known issue when urllib2 just skips the other part of provided
# url splitted with space char while urlencoding it in the later phase
url = url.replace(" ", "%20")
code = None
page = None
requestMsg = u"HTTP request [#%d]:\n%s " % (threadData.lastRequestUID, method or (HTTPMETHOD.POST if post else HTTPMETHOD.GET))
requestMsg += "%s" % urlparse.urlsplit(url)[2] or "/"
responseMsg = u"HTTP response "
requestHeaders = u""
responseHeaders = None
logHeaders = u""
skipLogTraffic = False
raise404 = raise404 and not kb.ignoreNotFound
# support for non-latin (e.g. cyrillic) URLs as urllib/urllib2 doesn't
# support those by default
url = asciifyUrl(url)
# fix for known issues when using url in unicode format
# (e.g. UnicodeDecodeError: "url = url + '?' + query" in redirect case)
url = unicodeencode(url)
try:
if silent:
socket.setdefaulttimeout(HTTP_SILENT_TIMEOUT)
else:
socket.setdefaulttimeout(conf.timeout)
if direct:
if "?" in url:
url, params = url.split("?")
params = urlencode(params)
url = "%s?%s" % (url, params)
requestMsg += "?%s" % params
elif multipart:
# Needed in this form because of potential circle dependency
# problem (option -> update -> connect -> option)
from lib.core.option import proxyHandler
multipartOpener = urllib2.build_opener(proxyHandler, multipartpost.MultipartPostHandler)
conn = multipartOpener.open(unicodeencode(url), multipart)
page = Connect.__connReadProxy(conn)
responseHeaders = conn.info()
responseHeaders[URI_HTTP_HEADER] = conn.geturl()
page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
return page
elif any ([refreshing, crawling]):
pass
elif target:
if conf.parameters.has_key(PLACE.GET) and not get:
get = conf.parameters[PLACE.GET]
if get:
url = "%s?%s" % (url, get)
requestMsg += "?%s" % get
if conf.method == HTTPMETHOD.POST and not post:
for place in (PLACE.POST, PLACE.SOAP):
if conf.parameters.has_key(place):
post = conf.parameters[place]
break
elif get:
url = "%s?%s" % (url, get)
requestMsg += "?%s" % get
requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str
# Prepare HTTP headers
headers = forgeHeaders({ HTTPHEADER.COOKIE: cookie, HTTPHEADER.USER_AGENT: ua, HTTPHEADER.REFERER: referer })
if conf.realTest:
headers[HTTPHEADER.REFERER] = "%s://%s" % (conf.scheme, conf.hostname)
if kb.authHeader:
headers[HTTPHEADER.AUTHORIZATION] = kb.authHeader
if kb.proxyAuthHeader:
headers[HTTPHEADER.PROXY_AUTHORIZATION] = kb.proxyAuthHeader
headers[HTTPHEADER.ACCEPT] = HTTP_ACCEPT_HEADER_VALUE
headers[HTTPHEADER.HOST] = host or getHostHeader(url)
if auxHeaders:
for key, item in auxHeaders.items():
headers[key] = item
for key, item in headers.items():
del headers[key]
headers[unicodeencode(key, kb.pageEncoding)] = unicodeencode(item, kb.pageEncoding)
post = unicodeencode(post, kb.pageEncoding)
if method:
req = MethodRequest(url, post, headers)
req.set_method(method)
else:
req = urllib2.Request(url, post, headers)
if not req.has_header(HTTPHEADER.ACCEPT_ENCODING):
requestHeaders += "%s: identity\n" % HTTPHEADER.ACCEPT_ENCODING
requestHeaders += "\n".join("%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in req.header_items())
if not req.has_header(HTTPHEADER.COOKIE) and conf.cj:
conf.cj._policy._now = conf.cj._now = int(time.time())
cookies = conf.cj._cookies_for_request(req)
requestHeaders += "\n%s" % ("Cookie: %s" % ";".join("%s=%s" % (getUnicode(cookie.name), getUnicode(cookie.value)) for cookie in cookies))
if not req.has_header(HTTPHEADER.CONNECTION):
requestHeaders += "\n%s: close" % HTTPHEADER.CONNECTION
requestMsg += "\n%s" % requestHeaders
if post:
requestMsg += "\n\n%s" % getUnicode(post)
requestMsg += "\n"
threadData.lastRequestMsg = requestMsg
logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg)
conn = urllib2.urlopen(req)
if not kb.authHeader and req.has_header(HTTPHEADER.AUTHORIZATION):
kb.authHeader = req.get_header(HTTPHEADER.AUTHORIZATION)
if not kb.proxyAuthHeader and req.has_header(HTTPHEADER.PROXY_AUTHORIZATION):
kb.proxyAuthHeader = req.get_header(HTTPHEADER.PROXY_AUTHORIZATION)
# Return response object
if response:
return conn, None, None
# Get HTTP response
if hasattr(conn, 'redurl'):
page = threadData.lastRedirectMsg[1] if kb.redirectChoice == REDIRECTION.NO\
else Connect.__connReadProxy(conn)
skipLogTraffic = kb.redirectChoice == REDIRECTION.NO
code = conn.redcode
else:
page = Connect.__connReadProxy(conn)
code = code or conn.code
responseHeaders = conn.info()
responseHeaders[URI_HTTP_HEADER] = conn.geturl()
page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
status = getUnicode(conn.msg)
if extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE) and not refreshing:
url = extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE)
debugMsg = "got HTML meta refresh header"
logger.debug(debugMsg)
if kb.alwaysRefresh is None:
msg = "sqlmap got a refresh request "
msg += "(redirect like response common to login pages). "
msg += "Do you want to apply the refresh "
msg += "from now on (or stay on the original page)? [Y/n]"
choice = readInput(msg, default="Y")
kb.alwaysRefresh = choice not in ("n", "N")
if kb.alwaysRefresh:
if url.lower().startswith('http://'):
kwargs['url'] = url
else:
kwargs['url'] = conf.url[:conf.url.rfind('/')+1] + url
threadData.lastRedirectMsg = (threadData.lastRequestUID, page)
kwargs['refreshing'] = True
kwargs['get'] = None
kwargs['post'] = None
try:
return Connect.__getPageProxy(**kwargs)
except sqlmapSyntaxException:
pass
# Explicit closing of connection object
if not conf.keepAlive:
try:
if hasattr(conn.fp, '_sock'):
conn.fp._sock.close()
conn.close()
except Exception, msg:
warnMsg = "problem occured during connection closing ('%s')" % msg
logger.warn(warnMsg)
except urllib2.HTTPError, e:
page = None
responseHeaders = None
try:
page = e.read()
responseHeaders = e.info()
responseHeaders[URI_HTTP_HEADER] = e.geturl()
page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
except socket.timeout:
warnMsg = "connection timed out while trying "
warnMsg += "to get error page information (%d)" % e.code
logger.warn(warnMsg)
return None, None, None
except KeyboardInterrupt:
raise
except:
pass
finally:
page = page if isinstance(page, unicode) else getUnicode(page)
code = e.code
threadData.lastHTTPError = (threadData.lastRequestUID, code)
kb.httpErrorCodes[code] = kb.httpErrorCodes.get(code, 0) + 1
status = getUnicode(e.msg)
responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status)
if responseHeaders:
logHeaders = "\n".join("%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in responseHeaders.items())
logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page))
skipLogTraffic = True
if conf.verbose <= 5:
responseMsg += getUnicode(logHeaders)
elif conf.verbose > 5:
responseMsg += "%s\n\n%s\n" % (logHeaders, page)
logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)
if e.code == httplib.UNAUTHORIZED:
errMsg = "not authorized, try to provide right HTTP "
errMsg += "authentication type and valid credentials (%d)" % code
raise sqlmapConnectionException, errMsg
elif e.code == httplib.NOT_FOUND:
if raise404:
errMsg = "page not found (%d)" % code
raise sqlmapConnectionException, errMsg
else:
debugMsg = "page not found (%d)" % code
logger.debug(debugMsg)
processResponse(page, responseHeaders)
elif e.code == httplib.GATEWAY_TIMEOUT:
if ignoreTimeout:
return None, None, None
else:
warnMsg = "unable to connect to the target url (%d - %s)" % (e.code, httplib.responses[e.code])
if threadData.retriesCount < conf.retries and not kb.threadException and not conf.realTest:
warnMsg += ", sqlmap is going to retry the request"
logger.critical(warnMsg)
return Connect.__retryProxy(**kwargs)
elif kb.testMode:
logger.critical(warnMsg)
return None, None, None
else:
raise sqlmapConnectionException, warnMsg
else:
debugMsg = "got HTTP error code: %d (%s)" % (code, status)
logger.debug(debugMsg)
def getPage(**kwargs):
"""
This method connects to the target url or proxy and returns
the target url page content
"""
if conf.delay is not None and isinstance(
conf.delay, (int, float)) and conf.delay > 0:
time.sleep(conf.delay)
elif conf.cpuThrottle:
cpuThrottle(conf.cpuThrottle)
threadData = getCurrentThreadData()
threadData.lastRequestUID += 1
# dirty hack because urllib2 just skips the other part of provided url
# splitted with space char while urlencoding it in the later phase
url = kwargs.get('url', conf.url).replace(" ", "%20")
get = kwargs.get('get', None)
post = kwargs.get('post', None)
method = kwargs.get('method', None)
cookie = kwargs.get('cookie', None)
ua = kwargs.get('ua', None)
referer = kwargs.get('referer', None)
direct = kwargs.get('direct', False)
multipart = kwargs.get('multipart', False)
silent = kwargs.get('silent', False)
raise404 = kwargs.get('raise404', True)
auxHeaders = kwargs.get('auxHeaders', None)
response = kwargs.get('response', False)
ignoreTimeout = kwargs.get('ignoreTimeout', False)
refreshing = kwargs.get('refreshing', False)
page = ""
cookieStr = ""
requestMsg = "HTTP request [#%d]:\n%s " % (threadData.lastRequestUID,
conf.method)
requestMsg += "%s" % urlparse.urlsplit(url)[2] or "/"
responseMsg = "HTTP response "
requestHeaders = ""
responseHeaders = None
logHeaders = ""
try:
if silent:
socket.setdefaulttimeout(HTTP_SILENT_TIMEOUT)
if direct:
if "?" in url:
url, params = url.split("?")
params = urlencode(params)
url = "%s?%s" % (url, params)
requestMsg += "?%s" % params
elif multipart:
# Needed in this form because of potential circle dependency
# problem (option -> update -> connect -> option)
from lib.core.option import proxyHandler
multipartOpener = urllib2.build_opener(
proxyHandler, multipartpost.MultipartPostHandler)
conn = multipartOpener.open(url, multipart)
page = conn.read()
responseHeaders = conn.info()
responseHeaders[URI_HTTP_HEADER] = conn.geturl()
page = decodePage(
page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING),
responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
return page
elif refreshing:
# Reference(s):
# http://vancouver-webpages.com/META/metatags.detail.html
# http://webdesign.about.com/od/metataglibraries/a/aa080300a.htm
get = None
post = None
else:
if conf.parameters.has_key(PLACE.GET) and not get:
get = conf.parameters[PLACE.GET]
if get:
url = "%s?%s" % (url, get)
requestMsg += "?%s" % get
if conf.method == HTTPMETHOD.POST:
if conf.parameters.has_key(PLACE.POST) and not post:
post = conf.parameters[PLACE.POST]
requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str
# Perform HTTP request
headers = forgeHeaders(cookie, ua, referer)
if conf.realTest:
headers[HTTPHEADER.REFERER] = "%s://%s" % (conf.scheme,
conf.hostname)
if kb.authHeader:
headers[HTTPHEADER.AUTHORIZATION] = kb.authHeader
if kb.proxyAuthHeader:
headers[HTTPHEADER.PROXY_AUTHORIZATION] = kb.proxyAuthHeader
if auxHeaders:
for key, item in auxHeaders.items():
headers[key] = item
for key, item in headers.items():
del headers[key]
headers[encodeUnicode(key, kb.pageEncoding)] = encodeUnicode(
item, kb.pageEncoding)
post = encodeUnicode(post, kb.pageEncoding)
if method:
req = MethodRequest(url, post, headers)
req.set_method(method)
else:
req = urllib2.Request(url, post, headers)
if not conf.dropSetCookie and conf.cj:
for _, cookie in enumerate(conf.cj):
if not cookieStr:
cookieStr = "Cookie: "
cookie = getUnicode(cookie)
index = cookie.index(" for ")
cookieStr += "%s; " % cookie[8:index]
if not req.has_header(HTTPHEADER.ACCEPT_ENCODING):
requestHeaders += "%s: identity\n" % HTTPHEADER.ACCEPT_ENCODING
requestHeaders += "\n".join([
"%s: %s" % (header, value)
for header, value in req.header_items()
])
if not req.has_header(HTTPHEADER.COOKIE) and cookieStr:
requestHeaders += "\n%s" % cookieStr[:-2]
if not req.has_header(HTTPHEADER.CONNECTION):
requestHeaders += "\n%s: close" % HTTPHEADER.CONNECTION
requestMsg += "\n%s" % requestHeaders
if post:
requestMsg += "\n\n%s" % post
requestMsg += "\n"
logger.log(8, requestMsg)
conn = urllib2.urlopen(req)
if not kb.authHeader and req.has_header(HTTPHEADER.AUTHORIZATION):
kb.authHeader = req.get_header(HTTPHEADER.AUTHORIZATION)
if not kb.proxyAuthHeader and req.has_header(
HTTPHEADER.PROXY_AUTHORIZATION):
kb.proxyAuthHeader = req.get_header(
HTTPHEADER.PROXY_AUTHORIZATION)
if hasattr(conn, "setcookie"):
kb.redirectSetCookie = conn.setcookie
if hasattr(conn, "redurl") and hasattr(
conn, "redcode"
) and not conf.redirectHandled and not conf.realTest:
msg = "sqlmap got a %d redirect to " % conn.redcode
msg += "%s - What target address do you " % conn.redurl
msg += "want to use from now on? %s " % conf.url
msg += "(default) or provide another target address based "
msg += "also on the redirection got from the application\n"
while True:
choice = readInput(msg, default=None)
if not choice:
pass
else:
conf.url = choice
try:
parseTargetUrl()
return Connect.__getPageProxy(**kwargs)
except sqlmapSyntaxException:
continue
break
conf.redirectHandled = True
# Reset the number of connection retries
kb.retriesCount = 0
# Return response object
if response:
return conn, None
# Get HTTP response
page = conn.read()
code = conn.code
responseHeaders = conn.info()
responseHeaders[URI_HTTP_HEADER] = conn.geturl()
page = decodePage(page,
responseHeaders.get(HTTPHEADER.CONTENT_ENCODING),
responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
status = getUnicode(conn.msg)
if extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL
| re.IGNORECASE) and not refreshing:
url = extractRegexResult(META_REFRESH_REGEX, page,
re.DOTALL | re.IGNORECASE)
if url.lower().startswith('http://'):
kwargs['url'] = url
else:
kwargs['url'] = conf.url[:conf.url.rfind('/') + 1] + url
threadData.lastRedirectMsg = (threadData.lastRequestUID, page)
kwargs['refreshing'] = True
debugMsg = "got HTML meta refresh header"
logger.debug(debugMsg)
try:
return Connect.__getPageProxy(**kwargs)
except sqlmapSyntaxException:
pass
# Explicit closing of connection object
if not conf.keepAlive:
try:
conn.fp._sock.close()
conn.close()
except Exception, msg:
warnMsg = "problem occured during connection closing ('%s')" % msg
logger.warn(warnMsg)
except urllib2.HTTPError, e:
page = None
responseHeaders = None
try:
page = e.read()
responseHeaders = e.info()
responseHeaders[URI_HTTP_HEADER] = e.geturl()
page = decodePage(
page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING),
responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
except socket.timeout:
warnMsg = "connection timed out while trying "
warnMsg += "to get error page information (%d)" % e.code
logger.warn(warnMsg)
return None, None
except:
pass
code = e.code
threadData.lastHTTPError = (threadData.lastRequestUID, code)
if code not in kb.httpErrorCodes:
kb.httpErrorCodes[code] = 0
kb.httpErrorCodes[code] += 1
status = getUnicode(e.msg)
responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID,
code, status)
if responseHeaders:
logHeaders = "\n".join([
"%s: %s" %
(key.capitalize() if isinstance(key, basestring) else key,
getUnicode(value))
for (key, value) in responseHeaders.items()
])
logHTTPTraffic(
requestMsg, "%s%s\n\n%s" %
(responseMsg, logHeaders,
page if isinstance(page, unicode) else getUnicode(page)))
if conf.verbose <= 5:
responseMsg += getUnicode(logHeaders)
elif conf.verbose > 5:
responseMsg += "%s\n\n%s\n" % (logHeaders, page)
logger.log(7, responseMsg)
if e.code == 401:
errMsg = "not authorized, try to provide right HTTP "
errMsg += "authentication type and valid credentials (%d)" % code
raise sqlmapConnectionException, errMsg
elif e.code == 404 and raise404:
errMsg = "page not found (%d)" % code
raise sqlmapConnectionException, errMsg
else:
debugMsg = "got HTTP error code: %d (%s)" % (code, status)
logger.debug(debugMsg)
page = processResponse(page, responseHeaders)
return page, responseHeaders