def getPage(**kwargs): """ This method connects to the target url or proxy and returns the target url page content """ if conf.delay is not None and isinstance(conf.delay, (int, float)) and conf.delay > 0: time.sleep(conf.delay) elif conf.cpuThrottle: delay = 0.00001 * (conf.cpuThrottle ** 2) time.sleep(delay) kb.locks.reqLock.acquire() kb.lastRequestUID += 1 requestID = kb.lastRequestUID kb.locks.reqLock.release() url = kwargs.get('url', conf.url).replace(" ", "%20") get = kwargs.get('get', None) post = kwargs.get('post', None) method = kwargs.get('method', None) cookie = kwargs.get('cookie', None) ua = kwargs.get('ua', None) direct = kwargs.get('direct', False) multipart = kwargs.get('multipart', False) silent = kwargs.get('silent', False) raise404 = kwargs.get('raise404', True) auxHeaders = kwargs.get('auxHeaders', None) response = kwargs.get('response', False) page = "" cookieStr = "" requestMsg = "HTTP request [#%d]:\n%s " % (requestID, conf.method) requestMsg += "%s" % urlparse.urlsplit(url)[2] or "/" responseMsg = "HTTP response " requestHeaders = "" responseHeaders = "" logHeaders = "" try: if silent: socket.setdefaulttimeout(3) if direct: if "?" in url: url, params = url.split("?") params = urlencode(params) url = "%s?%s" % (url, params) requestMsg += "?%s" % params elif multipart: # Needed in this form because of potential circle dependency # problem (option -> update -> connect -> option) from lib.core.option import proxyHandler multipartOpener = urllib2.build_opener(proxyHandler, multipartpost.MultipartPostHandler) conn = multipartOpener.open(url, multipart) page = conn.read() responseHeaders = conn.info() page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type")) return page else: if conf.parameters.has_key(PLACE.GET) and not get: get = conf.parameters[PLACE.GET] if get: url = "%s?%s" % (url, get) requestMsg += "?%s" % get if conf.method == HTTPMETHOD.POST: if conf.parameters.has_key(PLACE.POST) and not post: post = conf.parameters[PLACE.POST] requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str # Perform HTTP request headers = forgeHeaders(cookie, ua) if kb.authHeader: headers["Authorization"] = kb.authHeader if kb.proxyAuthHeader: headers["Proxy-authorization"] = kb.proxyAuthHeader if auxHeaders: for key, item in auxHeaders.items(): headers[key] = item if method: req = MethodRequest(url, post, headers) req.set_method(method) else: req = urllib2.Request(url, post, headers) if not conf.dropSetCookie and conf.cj: for _, cookie in enumerate(conf.cj): if not cookieStr: cookieStr = "Cookie: " cookie = getUnicode(cookie) index = cookie.index(" for ") cookieStr += "%s; " % cookie[8:index] conn = urllib2.urlopen(req) if not req.has_header("Accept-Encoding"): requestHeaders += "Accept-Encoding: identity\n" requestHeaders += "\n".join(["%s: %s" % (header, value) for header, value in req.header_items()]) if not req.has_header("Cookie") and cookieStr: requestHeaders += "\n%s" % cookieStr[:-2] if not req.has_header("Connection"): requestHeaders += "\nConnection: close" requestMsg += "\n%s" % requestHeaders if post: requestMsg += "\n%s" % post requestMsg += "\n" logger.log(8, requestMsg) if not kb.authHeader and req.has_header("Authorization"): kb.authHeader = req.get_header("Authorization") if not kb.proxyAuthHeader and req.has_header("Proxy-authorization"): kb.proxyAuthHeader = req.get_header("Proxy-authorization") if hasattr(conn, "redurl") and hasattr(conn, "redcode") and not conf.redirectHandled: msg = "sqlmap got a %d redirect to " % conn.redcode msg += "%s - What target address do you " % conn.redurl msg += "want to use from now on? %s " % conf.url msg += "(default) or provide another target address based " msg += "also on the redirection got from the application\n" while True: choice = readInput(msg, default="1") if not choice or choice == "1": pass else: conf.url = choice return Connect.__getPageProxy(**kwargs) break conf.redirectHandled = True # Reset the number of connection retries conf.retriesCount = 0 # Return response object if response: return conn, None # Get HTTP response page = conn.read() code = conn.code status = conn.msg responseHeaders = conn.info() page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type")) except urllib2.HTTPError, e: code = e.code status = e.msg try: page = e.read() responseHeaders = e.info() except socket.timeout: warnMsg = "connection timed out while trying " warnMsg += "to get error page information (%d)" % code logger.warn(warnMsg) return None, None except: pass responseMsg = "\n%s[#%d] (%d %s):\n" % (responseMsg, requestID, code, status) if responseHeaders: logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, value) for (key, value) in responseHeaders.items()]) logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page)) if e.code == 401: errMsg = "not authorized, try to provide right HTTP " errMsg += "authentication type and valid credentials (%d)" % code raise sqlmapConnectionException, errMsg elif e.code == 404 and raise404: errMsg = "page not found (%d)" % code raise sqlmapConnectionException, errMsg else: debugMsg = "got HTTP error code: %d (%s)" % (code, status) logger.debug(debugMsg)
def getPage(**kwargs): """ This method connects to the target url or proxy and returns the target url page content """ if conf.delay is not None and isinstance(conf.delay, (int, float)) and conf.delay > 0: time.sleep(conf.delay) elif conf.cpuThrottle: cpuThrottle(conf.cpuThrottle) threadData = getCurrentThreadData() threadData.lastRequestUID += 1 # dirty hack because urllib2 just skips the other part of provided url # splitted with space char while urlencoding it in the later phase url = kwargs.get('url', conf.url).replace(" ", "%20") get = kwargs.get('get', None) post = kwargs.get('post', None) method = kwargs.get('method', None) cookie = kwargs.get('cookie', None) ua = kwargs.get('ua', None) referer = kwargs.get('referer', None) direct = kwargs.get('direct', False) multipart = kwargs.get('multipart', False) silent = kwargs.get('silent', False) raise404 = kwargs.get('raise404', True) auxHeaders = kwargs.get('auxHeaders', None) response = kwargs.get('response', False) ignoreTimeout = kwargs.get('ignoreTimeout', False) refreshing = kwargs.get('refreshing', False) page = "" cookieStr = "" requestMsg = "HTTP request [#%d]:\n%s " % (threadData.lastRequestUID, conf.method) requestMsg += "%s" % urlparse.urlsplit(url)[2] or "/" responseMsg = "HTTP response " requestHeaders = "" responseHeaders = None logHeaders = "" try: if silent: socket.setdefaulttimeout(HTTP_SILENT_TIMEOUT) if direct: if "?" in url: url, params = url.split("?") params = urlencode(params) url = "%s?%s" % (url, params) requestMsg += "?%s" % params elif multipart: # Needed in this form because of potential circle dependency # problem (option -> update -> connect -> option) from lib.core.option import proxyHandler multipartOpener = urllib2.build_opener(proxyHandler, multipartpost.MultipartPostHandler) conn = multipartOpener.open(url, multipart) page = conn.read() responseHeaders = conn.info() responseHeaders[URI_HTTP_HEADER] = conn.geturl() page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE)) return page elif refreshing: # Reference(s): # http://vancouver-webpages.com/META/metatags.detail.html # http://webdesign.about.com/od/metataglibraries/a/aa080300a.htm get = None post = None else: if conf.parameters.has_key(PLACE.GET) and not get: get = conf.parameters[PLACE.GET] if get: url = "%s?%s" % (url, get) requestMsg += "?%s" % get if conf.method == HTTPMETHOD.POST: if conf.parameters.has_key(PLACE.POST) and not post: post = conf.parameters[PLACE.POST] requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str # Perform HTTP request headers = forgeHeaders(cookie, ua, referer) if conf.realTest: headers[HTTPHEADER.REFERER] = "%s://%s" % (conf.scheme, conf.hostname) if kb.authHeader: headers[HTTPHEADER.AUTHORIZATION] = kb.authHeader if kb.proxyAuthHeader: headers[HTTPHEADER.PROXY_AUTHORIZATION] = kb.proxyAuthHeader if auxHeaders: for key, item in auxHeaders.items(): headers[key] = item for key, item in headers.items(): del headers[key] headers[encodeUnicode(key, kb.pageEncoding)] = encodeUnicode(item, kb.pageEncoding) post = encodeUnicode(post, kb.pageEncoding) if method: req = MethodRequest(url, post, headers) req.set_method(method) else: req = urllib2.Request(url, post, headers) if not conf.dropSetCookie and conf.cj: for _, cookie in enumerate(conf.cj): if not cookieStr: cookieStr = "Cookie: " cookie = getUnicode(cookie) index = cookie.index(" for ") cookieStr += "%s; " % cookie[8:index] if not req.has_header(HTTPHEADER.ACCEPT_ENCODING): requestHeaders += "%s: identity\n" % HTTPHEADER.ACCEPT_ENCODING requestHeaders += "\n".join(["%s: %s" % (header, value) for header, value in req.header_items()]) if not req.has_header(HTTPHEADER.COOKIE) and cookieStr: requestHeaders += "\n%s" % cookieStr[:-2] if not req.has_header(HTTPHEADER.CONNECTION): requestHeaders += "\n%s: close" % HTTPHEADER.CONNECTION requestMsg += "\n%s" % requestHeaders if post: requestMsg += "\n\n%s" % post requestMsg += "\n" logger.log(8, requestMsg) conn = urllib2.urlopen(req) if not kb.authHeader and req.has_header(HTTPHEADER.AUTHORIZATION): kb.authHeader = req.get_header(HTTPHEADER.AUTHORIZATION) if not kb.proxyAuthHeader and req.has_header(HTTPHEADER.PROXY_AUTHORIZATION): kb.proxyAuthHeader = req.get_header(HTTPHEADER.PROXY_AUTHORIZATION) if hasattr(conn, "setcookie"): kb.redirectSetCookie = conn.setcookie if hasattr(conn, "redurl") and hasattr(conn, "redcode") and not conf.redirectHandled and not conf.realTest: msg = "sqlmap got a %d redirect to " % conn.redcode msg += "%s - What target address do you " % conn.redurl msg += "want to use from now on? %s " % conf.url msg += "(default) or provide another target address based " msg += "also on the redirection got from the application\n" while True: choice = readInput(msg, default=None) if not choice: pass else: conf.url = choice try: parseTargetUrl() return Connect.__getPageProxy(**kwargs) except sqlmapSyntaxException: continue break conf.redirectHandled = True # Reset the number of connection retries kb.retriesCount = 0 # Return response object if response: return conn, None # Get HTTP response page = conn.read() code = conn.code responseHeaders = conn.info() responseHeaders[URI_HTTP_HEADER] = conn.geturl() page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE)) status = getUnicode(conn.msg) if extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE) and not refreshing: url = extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE) if url.lower().startswith('http://'): kwargs['url'] = url else: kwargs['url'] = conf.url[:conf.url.rfind('/')+1] + url threadData.lastRedirectMsg = (threadData.lastRequestUID, page) kwargs['refreshing'] = True debugMsg = "got HTML meta refresh header" logger.debug(debugMsg) try: return Connect.__getPageProxy(**kwargs) except sqlmapSyntaxException: pass # Explicit closing of connection object if not conf.keepAlive: try: conn.fp._sock.close() conn.close() except Exception, msg: warnMsg = "problem occured during connection closing ('%s')" % msg logger.warn(warnMsg) except urllib2.HTTPError, e: page = None responseHeaders = None try: page = e.read() responseHeaders = e.info() responseHeaders[URI_HTTP_HEADER] = e.geturl() page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE)) except socket.timeout: warnMsg = "connection timed out while trying " warnMsg += "to get error page information (%d)" % e.code logger.warn(warnMsg) return None, None except: pass code = e.code threadData.lastHTTPError = (threadData.lastRequestUID, code) if code not in kb.httpErrorCodes: kb.httpErrorCodes[code] = 0 kb.httpErrorCodes[code] += 1 status = getUnicode(e.msg) responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status) if responseHeaders: logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in responseHeaders.items()]) logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page if isinstance(page, unicode) else getUnicode(page))) if conf.verbose <= 5: responseMsg += getUnicode(logHeaders) elif conf.verbose > 5: responseMsg += "%s\n\n%s\n" % (logHeaders, page) logger.log(7, responseMsg) if e.code == 401: errMsg = "not authorized, try to provide right HTTP " errMsg += "authentication type and valid credentials (%d)" % code raise sqlmapConnectionException, errMsg elif e.code == 404 and raise404: errMsg = "page not found (%d)" % code raise sqlmapConnectionException, errMsg else: debugMsg = "got HTTP error code: %d (%s)" % (code, status) logger.debug(debugMsg) page = processResponse(page, responseHeaders) return page, responseHeaders
def getPage(**kwargs): """ This method connects to the target url or proxy and returns the target url page content """ if conf.delay is not None and isinstance(conf.delay, (int, float)) and conf.delay > 0: time.sleep(conf.delay) elif conf.cpuThrottle: cpuThrottle(conf.cpuThrottle) threadData = getCurrentThreadData() threadData.lastRequestUID += 1 url = kwargs.get('url', conf.url) get = kwargs.get('get', None) post = kwargs.get('post', None) method = kwargs.get('method', None) cookie = kwargs.get('cookie', None) ua = kwargs.get('ua', None) referer = kwargs.get('referer', None) host = kwargs.get('host', conf.host) direct = kwargs.get('direct', False) multipart = kwargs.get('multipart', False) silent = kwargs.get('silent', False) raise404 = kwargs.get('raise404', True) auxHeaders = kwargs.get('auxHeaders', None) response = kwargs.get('response', False) ignoreTimeout = kwargs.get('ignoreTimeout', kb.ignoreTimeout) refreshing = kwargs.get('refreshing', False) retrying = kwargs.get('retrying', False) crawling = kwargs.get('crawling', False) if not urlparse.urlsplit(url).netloc: url = urlparse.urljoin(conf.url, url) # flag to know if we are dealing with the same target host target = reduce(lambda x, y: x == y, map(lambda x: urlparse.urlparse(x).netloc.split(':')[0], [url, conf.url or ""])) if not retrying: # Reset the number of connection retries threadData.retriesCount = 0 # fix for known issue when urllib2 just skips the other part of provided # url splitted with space char while urlencoding it in the later phase url = url.replace(" ", "%20") code = None page = None requestMsg = u"HTTP request [#%d]:\n%s " % (threadData.lastRequestUID, method or (HTTPMETHOD.POST if post else HTTPMETHOD.GET)) requestMsg += "%s" % urlparse.urlsplit(url)[2] or "/" responseMsg = u"HTTP response " requestHeaders = u"" responseHeaders = None logHeaders = u"" skipLogTraffic = False raise404 = raise404 and not kb.ignoreNotFound # support for non-latin (e.g. cyrillic) URLs as urllib/urllib2 doesn't # support those by default url = asciifyUrl(url) # fix for known issues when using url in unicode format # (e.g. UnicodeDecodeError: "url = url + '?' + query" in redirect case) url = unicodeencode(url) try: if silent: socket.setdefaulttimeout(HTTP_SILENT_TIMEOUT) else: socket.setdefaulttimeout(conf.timeout) if direct: if "?" in url: url, params = url.split("?") params = urlencode(params) url = "%s?%s" % (url, params) requestMsg += "?%s" % params elif multipart: # Needed in this form because of potential circle dependency # problem (option -> update -> connect -> option) from lib.core.option import proxyHandler multipartOpener = urllib2.build_opener(proxyHandler, multipartpost.MultipartPostHandler) conn = multipartOpener.open(unicodeencode(url), multipart) page = Connect.__connReadProxy(conn) responseHeaders = conn.info() responseHeaders[URI_HTTP_HEADER] = conn.geturl() page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE)) return page elif any ([refreshing, crawling]): pass elif target: if conf.parameters.has_key(PLACE.GET) and not get: get = conf.parameters[PLACE.GET] if get: url = "%s?%s" % (url, get) requestMsg += "?%s" % get if conf.method == HTTPMETHOD.POST and not post: for place in (PLACE.POST, PLACE.SOAP): if conf.parameters.has_key(place): post = conf.parameters[place] break elif get: url = "%s?%s" % (url, get) requestMsg += "?%s" % get requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str # Prepare HTTP headers headers = forgeHeaders({ HTTPHEADER.COOKIE: cookie, HTTPHEADER.USER_AGENT: ua, HTTPHEADER.REFERER: referer }) if conf.realTest: headers[HTTPHEADER.REFERER] = "%s://%s" % (conf.scheme, conf.hostname) if kb.authHeader: headers[HTTPHEADER.AUTHORIZATION] = kb.authHeader if kb.proxyAuthHeader: headers[HTTPHEADER.PROXY_AUTHORIZATION] = kb.proxyAuthHeader headers[HTTPHEADER.ACCEPT] = HTTP_ACCEPT_HEADER_VALUE headers[HTTPHEADER.HOST] = host or getHostHeader(url) if auxHeaders: for key, item in auxHeaders.items(): headers[key] = item for key, item in headers.items(): del headers[key] headers[unicodeencode(key, kb.pageEncoding)] = unicodeencode(item, kb.pageEncoding) post = unicodeencode(post, kb.pageEncoding) if method: req = MethodRequest(url, post, headers) req.set_method(method) else: req = urllib2.Request(url, post, headers) if not req.has_header(HTTPHEADER.ACCEPT_ENCODING): requestHeaders += "%s: identity\n" % HTTPHEADER.ACCEPT_ENCODING requestHeaders += "\n".join("%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in req.header_items()) if not req.has_header(HTTPHEADER.COOKIE) and conf.cj: conf.cj._policy._now = conf.cj._now = int(time.time()) cookies = conf.cj._cookies_for_request(req) requestHeaders += "\n%s" % ("Cookie: %s" % ";".join("%s=%s" % (getUnicode(cookie.name), getUnicode(cookie.value)) for cookie in cookies)) if not req.has_header(HTTPHEADER.CONNECTION): requestHeaders += "\n%s: close" % HTTPHEADER.CONNECTION requestMsg += "\n%s" % requestHeaders if post: requestMsg += "\n\n%s" % getUnicode(post) requestMsg += "\n" threadData.lastRequestMsg = requestMsg logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg) conn = urllib2.urlopen(req) if not kb.authHeader and req.has_header(HTTPHEADER.AUTHORIZATION): kb.authHeader = req.get_header(HTTPHEADER.AUTHORIZATION) if not kb.proxyAuthHeader and req.has_header(HTTPHEADER.PROXY_AUTHORIZATION): kb.proxyAuthHeader = req.get_header(HTTPHEADER.PROXY_AUTHORIZATION) # Return response object if response: return conn, None, None # Get HTTP response if hasattr(conn, 'redurl'): page = threadData.lastRedirectMsg[1] if kb.redirectChoice == REDIRECTION.NO\ else Connect.__connReadProxy(conn) skipLogTraffic = kb.redirectChoice == REDIRECTION.NO code = conn.redcode else: page = Connect.__connReadProxy(conn) code = code or conn.code responseHeaders = conn.info() responseHeaders[URI_HTTP_HEADER] = conn.geturl() page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE)) status = getUnicode(conn.msg) if extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE) and not refreshing: url = extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE) debugMsg = "got HTML meta refresh header" logger.debug(debugMsg) if kb.alwaysRefresh is None: msg = "sqlmap got a refresh request " msg += "(redirect like response common to login pages). " msg += "Do you want to apply the refresh " msg += "from now on (or stay on the original page)? [Y/n]" choice = readInput(msg, default="Y") kb.alwaysRefresh = choice not in ("n", "N") if kb.alwaysRefresh: if url.lower().startswith('http://'): kwargs['url'] = url else: kwargs['url'] = conf.url[:conf.url.rfind('/')+1] + url threadData.lastRedirectMsg = (threadData.lastRequestUID, page) kwargs['refreshing'] = True kwargs['get'] = None kwargs['post'] = None try: return Connect.__getPageProxy(**kwargs) except sqlmapSyntaxException: pass # Explicit closing of connection object if not conf.keepAlive: try: if hasattr(conn.fp, '_sock'): conn.fp._sock.close() conn.close() except Exception, msg: warnMsg = "problem occured during connection closing ('%s')" % msg logger.warn(warnMsg) except urllib2.HTTPError, e: page = None responseHeaders = None try: page = e.read() responseHeaders = e.info() responseHeaders[URI_HTTP_HEADER] = e.geturl() page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE)) except socket.timeout: warnMsg = "connection timed out while trying " warnMsg += "to get error page information (%d)" % e.code logger.warn(warnMsg) return None, None, None except KeyboardInterrupt: raise except: pass finally: page = page if isinstance(page, unicode) else getUnicode(page) code = e.code threadData.lastHTTPError = (threadData.lastRequestUID, code) kb.httpErrorCodes[code] = kb.httpErrorCodes.get(code, 0) + 1 status = getUnicode(e.msg) responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status) if responseHeaders: logHeaders = "\n".join("%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in responseHeaders.items()) logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page)) skipLogTraffic = True if conf.verbose <= 5: responseMsg += getUnicode(logHeaders) elif conf.verbose > 5: responseMsg += "%s\n\n%s\n" % (logHeaders, page) logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg) if e.code == httplib.UNAUTHORIZED: errMsg = "not authorized, try to provide right HTTP " errMsg += "authentication type and valid credentials (%d)" % code raise sqlmapConnectionException, errMsg elif e.code == httplib.NOT_FOUND: if raise404: errMsg = "page not found (%d)" % code raise sqlmapConnectionException, errMsg else: debugMsg = "page not found (%d)" % code logger.debug(debugMsg) processResponse(page, responseHeaders) elif e.code == httplib.GATEWAY_TIMEOUT: if ignoreTimeout: return None, None, None else: warnMsg = "unable to connect to the target url (%d - %s)" % (e.code, httplib.responses[e.code]) if threadData.retriesCount < conf.retries and not kb.threadException and not conf.realTest: warnMsg += ", sqlmap is going to retry the request" logger.critical(warnMsg) return Connect.__retryProxy(**kwargs) elif kb.testMode: logger.critical(warnMsg) return None, None, None else: raise sqlmapConnectionException, warnMsg else: debugMsg = "got HTTP error code: %d (%s)" % (code, status) logger.debug(debugMsg)
def getPage(**kwargs): """ This method connects to the target url or proxy and returns the target url page content """ if conf.delay is not None and isinstance( conf.delay, (int, float)) and conf.delay > 0: time.sleep(conf.delay) elif conf.cpuThrottle: cpuThrottle(conf.cpuThrottle) threadData = getCurrentThreadData() threadData.lastRequestUID += 1 # dirty hack because urllib2 just skips the other part of provided url # splitted with space char while urlencoding it in the later phase url = kwargs.get('url', conf.url).replace(" ", "%20") get = kwargs.get('get', None) post = kwargs.get('post', None) method = kwargs.get('method', None) cookie = kwargs.get('cookie', None) ua = kwargs.get('ua', None) referer = kwargs.get('referer', None) direct = kwargs.get('direct', False) multipart = kwargs.get('multipart', False) silent = kwargs.get('silent', False) raise404 = kwargs.get('raise404', True) auxHeaders = kwargs.get('auxHeaders', None) response = kwargs.get('response', False) ignoreTimeout = kwargs.get('ignoreTimeout', False) refreshing = kwargs.get('refreshing', False) page = "" cookieStr = "" requestMsg = "HTTP request [#%d]:\n%s " % (threadData.lastRequestUID, conf.method) requestMsg += "%s" % urlparse.urlsplit(url)[2] or "/" responseMsg = "HTTP response " requestHeaders = "" responseHeaders = None logHeaders = "" try: if silent: socket.setdefaulttimeout(HTTP_SILENT_TIMEOUT) if direct: if "?" in url: url, params = url.split("?") params = urlencode(params) url = "%s?%s" % (url, params) requestMsg += "?%s" % params elif multipart: # Needed in this form because of potential circle dependency # problem (option -> update -> connect -> option) from lib.core.option import proxyHandler multipartOpener = urllib2.build_opener( proxyHandler, multipartpost.MultipartPostHandler) conn = multipartOpener.open(url, multipart) page = conn.read() responseHeaders = conn.info() responseHeaders[URI_HTTP_HEADER] = conn.geturl() page = decodePage( page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE)) return page elif refreshing: # Reference(s): # http://vancouver-webpages.com/META/metatags.detail.html # http://webdesign.about.com/od/metataglibraries/a/aa080300a.htm get = None post = None else: if conf.parameters.has_key(PLACE.GET) and not get: get = conf.parameters[PLACE.GET] if get: url = "%s?%s" % (url, get) requestMsg += "?%s" % get if conf.method == HTTPMETHOD.POST: if conf.parameters.has_key(PLACE.POST) and not post: post = conf.parameters[PLACE.POST] requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str # Perform HTTP request headers = forgeHeaders(cookie, ua, referer) if conf.realTest: headers[HTTPHEADER.REFERER] = "%s://%s" % (conf.scheme, conf.hostname) if kb.authHeader: headers[HTTPHEADER.AUTHORIZATION] = kb.authHeader if kb.proxyAuthHeader: headers[HTTPHEADER.PROXY_AUTHORIZATION] = kb.proxyAuthHeader if auxHeaders: for key, item in auxHeaders.items(): headers[key] = item for key, item in headers.items(): del headers[key] headers[encodeUnicode(key, kb.pageEncoding)] = encodeUnicode( item, kb.pageEncoding) post = encodeUnicode(post, kb.pageEncoding) if method: req = MethodRequest(url, post, headers) req.set_method(method) else: req = urllib2.Request(url, post, headers) if not conf.dropSetCookie and conf.cj: for _, cookie in enumerate(conf.cj): if not cookieStr: cookieStr = "Cookie: " cookie = getUnicode(cookie) index = cookie.index(" for ") cookieStr += "%s; " % cookie[8:index] if not req.has_header(HTTPHEADER.ACCEPT_ENCODING): requestHeaders += "%s: identity\n" % HTTPHEADER.ACCEPT_ENCODING requestHeaders += "\n".join([ "%s: %s" % (header, value) for header, value in req.header_items() ]) if not req.has_header(HTTPHEADER.COOKIE) and cookieStr: requestHeaders += "\n%s" % cookieStr[:-2] if not req.has_header(HTTPHEADER.CONNECTION): requestHeaders += "\n%s: close" % HTTPHEADER.CONNECTION requestMsg += "\n%s" % requestHeaders if post: requestMsg += "\n\n%s" % post requestMsg += "\n" logger.log(8, requestMsg) conn = urllib2.urlopen(req) if not kb.authHeader and req.has_header(HTTPHEADER.AUTHORIZATION): kb.authHeader = req.get_header(HTTPHEADER.AUTHORIZATION) if not kb.proxyAuthHeader and req.has_header( HTTPHEADER.PROXY_AUTHORIZATION): kb.proxyAuthHeader = req.get_header( HTTPHEADER.PROXY_AUTHORIZATION) if hasattr(conn, "setcookie"): kb.redirectSetCookie = conn.setcookie if hasattr(conn, "redurl") and hasattr( conn, "redcode" ) and not conf.redirectHandled and not conf.realTest: msg = "sqlmap got a %d redirect to " % conn.redcode msg += "%s - What target address do you " % conn.redurl msg += "want to use from now on? %s " % conf.url msg += "(default) or provide another target address based " msg += "also on the redirection got from the application\n" while True: choice = readInput(msg, default=None) if not choice: pass else: conf.url = choice try: parseTargetUrl() return Connect.__getPageProxy(**kwargs) except sqlmapSyntaxException: continue break conf.redirectHandled = True # Reset the number of connection retries kb.retriesCount = 0 # Return response object if response: return conn, None # Get HTTP response page = conn.read() code = conn.code responseHeaders = conn.info() responseHeaders[URI_HTTP_HEADER] = conn.geturl() page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE)) status = getUnicode(conn.msg) if extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE) and not refreshing: url = extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE) if url.lower().startswith('http://'): kwargs['url'] = url else: kwargs['url'] = conf.url[:conf.url.rfind('/') + 1] + url threadData.lastRedirectMsg = (threadData.lastRequestUID, page) kwargs['refreshing'] = True debugMsg = "got HTML meta refresh header" logger.debug(debugMsg) try: return Connect.__getPageProxy(**kwargs) except sqlmapSyntaxException: pass # Explicit closing of connection object if not conf.keepAlive: try: conn.fp._sock.close() conn.close() except Exception, msg: warnMsg = "problem occured during connection closing ('%s')" % msg logger.warn(warnMsg) except urllib2.HTTPError, e: page = None responseHeaders = None try: page = e.read() responseHeaders = e.info() responseHeaders[URI_HTTP_HEADER] = e.geturl() page = decodePage( page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE)) except socket.timeout: warnMsg = "connection timed out while trying " warnMsg += "to get error page information (%d)" % e.code logger.warn(warnMsg) return None, None except: pass code = e.code threadData.lastHTTPError = (threadData.lastRequestUID, code) if code not in kb.httpErrorCodes: kb.httpErrorCodes[code] = 0 kb.httpErrorCodes[code] += 1 status = getUnicode(e.msg) responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status) if responseHeaders: logHeaders = "\n".join([ "%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in responseHeaders.items() ]) logHTTPTraffic( requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page if isinstance(page, unicode) else getUnicode(page))) if conf.verbose <= 5: responseMsg += getUnicode(logHeaders) elif conf.verbose > 5: responseMsg += "%s\n\n%s\n" % (logHeaders, page) logger.log(7, responseMsg) if e.code == 401: errMsg = "not authorized, try to provide right HTTP " errMsg += "authentication type and valid credentials (%d)" % code raise sqlmapConnectionException, errMsg elif e.code == 404 and raise404: errMsg = "page not found (%d)" % code raise sqlmapConnectionException, errMsg else: debugMsg = "got HTTP error code: %d (%s)" % (code, status) logger.debug(debugMsg) page = processResponse(page, responseHeaders) return page, responseHeaders