def process_http(raw): headers, body = raw.split(BODY_SEP, 1) if headers.startswith('CONNECT'): return None if not len(body): return { 'headers': headers, 'body': None } source = FakeSocket(raw) response = HTTPResponse(source) response.begin() body = None if response.getheader('Content-Encoding') == 'gzip': buf = StringIO(response.read(len(raw))) f = gzip.GzipFile(fileobj=buf) body = f.read() else: body = response.read(len(raw)) return { 'headers': headers, 'body': body if len(body) else None }
def setUp(self): from httplib import HTTPResponse with Stub() as HTTPResponse: HTTPResponse.status >> 200 HTTPResponse.read() >> '{"id": "THE-PRECIOUS-GUID", "title": "example title", "subtitle": "example title - subtitle", "description": "a description", "user": "******", "tags": [], "created_at": 1329767353.0, "source": "http://www.example.com/example.csv", "link": "http://www.junar.com/datastreams/some-url"}' with Stub() as conn: from httplib import HTTPConnection conn = HTTPConnection("api.junar.com") conn.request("POST", "/datastreams/publish", self.params, self.headers) conn.getresponse() >> HTTPResponse
def get_json(self, response): res_str = str(helpers.bytesToString(response.getResponse())) source = FakeSocket(res_str) res_obj = HTTPResponse(source) res_obj.begin() print "Reading response object" return json.loads(res_obj.read())
def send_request(self, request): self.request_num += 1 # Sanity check: if we're sending a request with a content-length but # we don't have that many bytes to send, we'll just get a 504. Don't # send it and instead report a client error. parts = request.split('\r\n\r\n', 1) if len(parts) > 1: req, body = parts match = content_length_re.search(req) if match: if len(body) < int(match.group(1)): self.error("request body of incorrect size") return True if not self.connection: self._connect() if self.connection: # tally request method #self.tally(request.split(" ", 1)[0]) try: self.connection.sendall(request) response = HTTPResponse(self.connection) response.begin() self.tally(response.status) while response.read(): pass if response.will_close: # We hope our Connection: keep-alive won't be ignored, but # deal with it if it does. self._disconnect() if self.options.speedup < 0.8: # if we're slowing down by too much, keep-alive will just # result in the server getting bored between requests and # dropping the connection, so disable it. self._disconnect() return True except IncompleteRead: self.error("error while reading response: IncompleteRead (terminating job)") self._disconnect() except Exception, e: # TODO: more restrictive error catching? self.error("error while sending request and reading response: %s %s" % (type(e), e)) self._disconnect() if self.connection: self.connection.close() self.connection = None
def sendAndReceive(self): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((ORIGINAL_IP, 80)) sock.sendall(str(self)) except: return GlobalTools.FailedHTTPResponse(), '' res = HTTPResponse(sock) res.begin() res_body = res.read() res.close() if 'transfer-encoding' in res.msg: # httplib.HTTPResponse automatically concatenate chunked response # but do not delete 'transfer-encoding' header # so the header must be deleted res.msg.__delitem__('transfer-encoding') compmeth = res.msg.getheader('content-encoding', '').lower() if compmeth and compmeth.find('identity') != 0: # response body is compressed with some method offset = 0 if compmeth.find('gzip') != -1: # if body is gziped, header offset value is 47 # if not, offset value is 0 # this server does not support sdch... offset += 47 res_body = decompress(res_body, offset) res.msg['content-encoding'] = 'identity' return res, res_body
def send_and_recv(self): try: # because www.dream-pro.info is tlanslated to 127.0.0.1 using hosts' entry, # send message to www.dream-pro.info with socket.socket to make # http connection sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM) sock.connect((WWW_DREAM_PRO_INFO,80)) sock.sendall(str(self)) except: print 'SocketError' return res = HTTPResponse(sock) res.begin() res_body = res.read() res.close() if 'transfer-encoding' in res.msg: # httplib.HTTPResponse automatically concatenate chunked response # but do not delete 'transfer-encoding' header # so the header must be deleted res.msg.__delitem__('transfer-encoding') compmeth = res.msg.getheader('content-encoding','').lower() if compmeth and compmeth.find('identity') != 0 : # response body is compressed with some method offset = 0 if compmeth.find('gzip') != -1: # if body is gziped, header offset value is 47 # if not, offset value is 0 # this server does not support sdch... offset += 47 res_body = decompress(res_body,offset) res.msg['content-encoding'] = 'identity' return res, res_body
def process_record(self, record): content = None try: payload = record.payload.read() s = FakeSocket(payload) response = HTTPResponse(s) response.begin() status_code = response.status if status_code != 200: return content_type = response.getheader('Content-Type', '') if 'text/html' not in content_type: return headers = response.getheaders() content = response.read(len(payload)) except Exception: self.increment_counter('errors', 'process_record', 1) logging.error('Error processing record: {}', traceback.format_exc()) return if content is not None: content = content.strip() if not content: return for item in self.process_content(record.url, headers, content): yield item
def sogouProxy(self): if self.remote is None or self.lastHost != self.headers["Host"]: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.remote.settimeout(RemoteTimeout) self.remote.connect((proxy_host, proxy_port)) self.remote.sendall(self.requestline.encode('ascii').replace('baike.baidu.com', '220.181.111.247') + b"\r\n") # Add Sogou Verification Tags self.headers["X-Sogou-Auth"] = x_sogou_auth t = hex(int(time.time()))[2:].rstrip('L').zfill(8) self.headers["X-Sogou-Tag"] = calc_sogou_hash(t, self.headers['Host']) self.headers["X-Sogou-Timestamp"] = t headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n") self.remote.sendall(headerstr.encode('ascii') + b"\r\n") # Send Post data if self.command == 'POST': self.remote.sendall(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) response.begin() # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status.encode('ascii') + b'\r\n') hlist = [] for line in response.msg.headers: # Fixed multiple values of a same name if 'TRANSFER-ENCODING' not in line.upper(): hlist.append(line) self.wfile.write("".join(hlist) + b'\r\n') if self.command == "CONNECT" and response.status == 200: return self.transfer(self.remote, self.connection) else: while True: response_data = response.read(BufferSize) if not response_data: break self.wfile.write(response_data)
def testPipelining(self): # Tests the use of several requests issued at once. s = ("GET / HTTP/1.0\r\n" "Connection: %s\r\n" "Content-Length: %d\r\n" "\r\n" "%s") to_send = '' count = 25 for n in range(count): body = "Response #%d\r\n" % (n + 1) if n + 1 < count: conn = 'keep-alive' else: conn = 'close' to_send += s % (conn, len(body), body) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((LOCALHOST, self.port)) sock.send(to_send) for n in range(count): expect_body = "Response #%d\r\n" % (n + 1) response = ClientHTTPResponse(sock) response.begin() self.failUnlessEqual(int(response.status), 200) length = int(response.getheader('Content-Length', '0')) response_body = response.read(length) self.failUnlessEqual(length, len(response_body)) self.failUnlessEqual(response_body, expect_body)
def proxy(self): if self.remote is None or self.lastHost != self.headers['Host']: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.remote.connect((proxy_host, proxy_port)) self.remote.sendall(self.requestline.encode('ascii') + b'\r\n') # Add auth inf. self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(auth) #self.remote.sendall('Proxy-Authorization: Basic %s\r\n' % base64.b64encode(auth)) headerstr = str(self.headers).replace('\r\n', '\n').replace('\n', '\r\n') self.remote.sendall(headerstr.encode('ascii') + b"\r\n") # Send Post data if self.command == 'POST': self.remote.sendall(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) response.begin() # Reply to the browser status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason self.wfile.write(status.encode('ascii') + b'\r\n') hlist = [] for line in response.msg.headers: # Fixed multiple values of a same name if 'TRANSFER-ENCODING' not in line.upper(): hlist.append(line) self.wfile.write(''.join(hlist) + b'\r\n') if self.command == 'CONNECT' and response.status == 200: return self.transfer(self.remote, self.connection) else: while True: response_data = response.read(BufferSize) if not response_data: break self.wfile.write(response_data)
def sogouProxy(self): if self.remote is None or self.lastHost != self.headers["Host"]: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.remote.settimeout(RemoteTimeout) self.remote.connect((proxy_host, proxy_port)) self.remote.sendall(self.requestline.encode('ascii') + b"\r\n") # Add Sogou Verification Tags self.headers["X-Sogou-Auth"] = x_sogou_auth t = hex(int(time.time()))[2:].rstrip('L').zfill(8) self.headers["X-Sogou-Tag"] = calc_sogou_hash(t, self.headers['Host']) self.headers["X-Sogou-Timestamp"] = t headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n") self.remote.sendall(headerstr.encode('ascii') + b"\r\n") # Send Post data if self.command == 'POST': self.remote.sendall(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) response.begin() # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status.encode('ascii') + b'\r\n') hlist = [] for line in response.msg.headers: # Fixed multiple values of a same name if 'TRANSFER-ENCODING' not in line.upper(): hlist.append(line) self.wfile.write("".join(hlist) + b'\r\n') if self.command == "CONNECT" and response.status == 200: return self.transfer(self.remote, self.connection) else: while True: response_data = response.read(BufferSize) if not response_data: break self.wfile.write(response_data)
def sogouProxy(self): if self.headers["Host"].startswith('chrome_dcp_proxy_pac.cnbeining'): #Give a PAC file self.wfile.write("HTTP/1.1 200 OK".encode('ascii') + b'\r\n') hstr = '''Host: 127.0.0.1 function FindProxyForURL(url, host) { if (url.substring(0,5) == 'http:' && !isPlainHostName(host) && !shExpMatch(host, '*.local') && !isInNet(dnsResolve(host), '10.0.0.0', '255.0.0.0') && !isInNet(dnsResolve(host), '172.16.0.0', '255.240.0.0') && !isInNet(dnsResolve(host), '192.168.0.0', '255.255.0.0') && !isInNet(dnsResolve(host), '127.0.0.0', '255.255.255.0') ) return 'PROXY ''' + server_ip + ':' + str(server_port) + '''; DIRECT'; return 'DIRECT'; }''' self.wfile.write(hstr + b'\r\n') return if self.remote is None or self.lastHost != self.headers["Host"]: if PROXY_MODE == 'HTTPS': context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) context.verify_mode = ssl.CERT_REQUIRED context.check_hostname = True context.load_default_certs() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(RemoteTimeout) self.remote = context.wrap_socket(s, server_hostname='proxy.googlezip.net') self.remote.connect(('proxy.googlezip.net', 443)) else: #HTTP self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.remote.settimeout(RemoteTimeout) self.remote.connect(("compress.googlezip.net", 80)) self.remote.sendall(self.requestline.encode('ascii') + b"\r\n") # Add Verification Tags self.headers["Chrome-Proxy"] = get_google_header() headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n") self.remote.sendall(headerstr.encode('ascii') + b"\r\n") # Send Post data if self.command == 'POST': self.remote.sendall(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) response.begin() # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status.encode('ascii') + b'\r\n') hlist = [] for line in response.msg.headers: # Fixed multiple values of a same name if 'TRANSFER-ENCODING' not in line.upper(): hlist.append(line) self.wfile.write("".join(hlist) + b'\r\n') if self.command == "CONNECT": # NO HTTPS, as Chrome DCP does not allow HTTPS traffic return else: while True: response_data = response.read(BufferSize) if not response_data: break self.wfile.write(response_data)
def _get_response(self): # Parse response h = HTTPResponse(self._proxy_sock) h.begin() res_d = { 'status': h.status, 'reason': h.reason, 'req_version': self.request_version, 'msg': h.msg, 'trans_enc': (h.msg['Transfer-Encoding'] if 'Transfer-Encoding' in h.msg else '[NO-ENCODING]') } # Get rid of the pesky header del h.msg['Transfer-Encoding'] res = '%(req_version)s %(status)s %(reason)s\r\n%(msg)s\r\n' % res_d res_d['data'] = h.read() res_d['time'] = time() res += res_d['data'] # Let's close off the remote end h.close() self._proxy_sock.close() # Relay the message self.request.sendall(self.mitm_response(res)) print res return res_d
def send_and_recv(self): try: # because www.dream-pro.info is tlanslated to 127.0.0.1 using hosts' entry, # send message to www.dream-pro.info with socket.socket to make # http connection sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((WWW_DREAM_PRO_INFO, 80)) sock.sendall(str(self)) except: print 'SocketError' return res = HTTPResponse(sock) res.begin() res_body = res.read() res.close() if 'transfer-encoding' in res.msg: # httplib.HTTPResponse automatically concatenate chunked response # but do not delete 'transfer-encoding' header # so the header must be deleted res.msg.__delitem__('transfer-encoding') compmeth = res.msg.getheader('content-encoding', '').lower() if compmeth and compmeth.find('identity') != 0: # response body is compressed with some method offset = 0 if compmeth.find('gzip') != -1: # if body is gziped, header offset value is 47 # if not, offset value is 0 # this server does not support sdch... offset += 47 res_body = decompress(res_body, offset) res.msg['content-encoding'] = 'identity' return res, res_body
def do_proxy(self): try: if self.s == 0: self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.s.connect((proxy_host, proxy_port)) self.s.send(self.requestline.encode('ascii') + b'\r\n') # Add Sogou Verification Tags self.headers['X-Sogou-Auth'] = x_sogou_auth t = hex(int(time.time()))[2:].rstrip('L').zfill(8) self.headers['X-Sogou-Tag'] = calc_sogou_hash(t, self.headers['Host']) self.headers['X-Sogou-Timestamp'] = t self.s.send(str(self.headers).encode('ascii') + b'\r\n') # Send Post data if(self.command=='POST'): self.s.send(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.s, method=self.command, buffering=True) response.begin() # Reply to the browser status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason self.wfile.write(status.encode('ascii') + b'\r\n') h = '' for hh, vv in response.getheaders(): if hh.upper()!='TRANSFER-ENCODING': h += hh + ': ' + vv + '\r\n' self.wfile.write(h.encode('ascii') + b'\r\n') while True: response_data = response.read(8192) if(len(response_data) == 0): break self.wfile.write(response_data) except socket.error: print('socket error for ' + self.requestline)
def open_image2(url, buff_size=4096, connect_timeout=5, read_timeout=4, full_read_timeout=30, logger=logging): t = urlparse.urlparse(url) path = t.path or '/' if len(t.netloc.split(':')) == 2: HOST, PORT = t.netloc.split(':') else: HOST, PORT = t.netloc, 80 UA = ('Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36' ' (KHTML, like Gecko) Chrome/54.0.2840.87 Safari/537.36') s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(connect_timeout) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: s.connect((HOST, PORT)) except Exception as e: logger.exception(e) return None s.send("GET %s HTTP/1.1\r\n" % path) s.send("Host:%s\r\n" % HOST) s.send("Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n") s.send("User-Agent:%s\r\n" % UA) s.send('\r\n') data = [] begin = time.time() while 1: if time.time() - begin > full_read_timeout: logger.error('reach full_read_timeout') data = [] break try: msg = s.recv(buff_size) #print len(msg), len(data), time.time() - begin if len(msg) == 0: break data.append(msg) except socket.timeout as e: if e.args[0] == 'timed out': break else: logger.error('open image [%s] socket timeout: %s' % (url, str(e))) data = [] break except socket.error as e: logger.error('open image [%s] socket error: %s' % (url, str(e))) data = [] break s.shutdown(1) s.close() data = ''.join(data) if data: resp = HTTPResponse(FakeSocket(data)) resp.begin() if resp.status == 200: return resp.read(len(data)) return None
def get(self, request): # Format and pass along the initial request request = self.format_request(request) self.outsocket.sendall(request) # Wait for request while (True): r, w, x = select.select([self.insocket, self.outsocket], [], [], 0) if self.insocket in r: request = self.insocket.recv(buff_size) if request == "": print "Got end message from browser" self.kill() else: try: request = self.format_request(request) except MismatchedHost: print "Host changed. Getting new end socket" self.insocket.sendall("") self.kill() self.outsocket.sendall(request) if self.outsocket in r: httpRes = HTTPResponse(self.outsocket) response = "" try: httpRes.begin() headers = str(httpRes.msg) print headers content = httpRes.read() # TODO: Move below to format_response # Fix chunked header if headers.find("Transfer-Encoding: chunked") != -1: headers = headers.replace( "Transfer-Encoding: chunked\r\n", "") headers += "Content Length: " + str( len(content)) + "\r\n" if httpRes.version == 10: response += "HTTP/1.0 " elif httpRes.version == 11: response += "HTTP/1.1 " response += str(httpRes.status) + " " + str( httpRes.reason) + nl response += headers + nl response += content print response except BadStatusLine: self.kill() self.insocket.sendall(response) self.kill() if self.killed: self.end() return
def filter_malware(response_str): global apikey source = HTTPObject(response_str) http_response = HTTPResponse(source) http_response.begin() url = 'https://www.virustotal.com/vtapi/v2/file/report' content = http_response.read(len(response_str)) hash = hashlib.md5(content).hexdigest() params = {'apikey': apikey, 'resource': hash} response = requests.get(url, params=params) return response
def getBody(msg): ''' Get the entity body section of the http message ''' class FakeSocket(): def __init__(self, msg): self.file = StringIO(msg) def makefile(self, *args, **kwargs): return self.file source = FakeSocket(msg) rsp = HTTPResponse(source) rsp.begin() return rsp.read(len(msg))
def decompress_payload(payload): try: source = FakeSocket(payload) res = HTTPResponse(source) res.begin() result = zlib.decompress(res.read(), 16 + zlib.MAX_WBITS) except Exception as e: result = payload # try: # result = '.'.join(str(ord(c)) for c in payload) # except: # result = payload return result
def get(self, request): # Format and pass along the initial request request = self.format_request(request) self.outsocket.sendall(request) # Wait for request while(True): r,w,x = select.select([self.insocket, self.outsocket],[],[],0) if self.insocket in r: request = self.insocket.recv(buff_size) if request == "": print "Got end message from browser" self.kill() else: try: request = self.format_request(request) except MismatchedHost: print "Host changed. Getting new end socket" self.insocket.sendall("") self.kill() self.outsocket.sendall(request) if self.outsocket in r: httpRes = HTTPResponse(self.outsocket) response = "" try: httpRes.begin() headers = str(httpRes.msg) content = httpRes.read() if headers.find("Transfer-Encoding: chunked") != -1: headers = headers.replace("Transfer-Encoding: chunked\r\n", "") headers += "Content Length: " + str(len(content)) + "\r\n" if httpRes.version == 10: response += "HTTP/1.0 " elif httpRes.version == 11: response += "HTTP/1.1 " response += str(httpRes.status) + " " + str(httpRes.reason) + nl response += headers + nl response += content #print response except BadStatusLine: self.kill() self.insocket.sendall(response) self.kill() #''' if self.killed: self.end() return
def get(addr, path, debug=False): client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) client.connect(addr) client.send("GET {} HTTP/1.0\r\n\r\n".format(path)) resp_str = client.recv(65536) source = FakeSocket(resp_str) resp = HTTPResponse(source) resp.begin() if resp.status == 200: text = resp.read(len(resp_str)) data = json.loads(text) if debug: log.debug(data) return data return {}
def sendData(self, ip): """ 套接字通信 :param ip: :return: """ addr = (ip, 80) data = "GET / HTTP/1.1\r\nHost: " + self.args.domain + "\r\n" + "Connection: close" + "\r\n\r\n" recvdata = "" sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) res = {} res.setdefault("ip", ip) res.setdefault("status") res.setdefault("body") res.setdefault("server_header") res.setdefault("x_powered_by_header") res.setdefault("success") try: sock.connect(addr) sock.send(data) while True: buffer = sock.recv(1024) if not buffer: break recvdata += buffer response = HTTPResponse(FakeSocket(recvdata)) response.begin() # begin有什么用??? res["status"] = response.status if response.status == 200: # 这里就可以保存为html文件了 msg = ip + " seems done!!!" self.log.info(msg) res["body"] = response.read() res["server"] = response.getheader("Server", default="Known") res["x_powered_by"] = response.getheader("X-Powered-By", default="Known") if self.args.keyword: if self.args.keyword in res["body"]: res["success"] = True else: res["success"] = False else: res["success"] = True self.result.append(res) except Exception as err: self.log.error(err) res["success"] = False
def get_json(self, response): res_bytes = response.getResponse() # byte[] if not res_bytes: print "[!] Failed to get json for Response. Response has not bytes" return {} res_str = str(helpers.bytesToString(res_bytes)) source = FakeSocket(res_str) res_obj = HTTPResponse(source) res_obj.begin() # print "Reading response object" res_text = res_obj.read() try: res_data = json.loads(res_text) except Exception as e: print "[!] Failed to convert to json: " + str(res_text) res_data = {} return res_data
def testWithoutCRLF(self): # Tests the use of just newlines rather than CR/LFs. data = "Echo\nthis\r\nplease" s = ("GET / HTTP/1.0\n" "Connection: close\n" "Content-Length: %d\n" "\n" "%s") % (len(data), data) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((LOCALHOST, self.port)) sock.send(s) response = ClientHTTPResponse(sock) response.begin() self.failUnlessEqual(int(response.status), 200) length = int(response.getheader('Content-Length', '0')) response_body = response.read(length) self.failUnlessEqual(length, len(data)) self.failUnlessEqual(response_body, data)
def isValidData(self, data): print(MESSAGES.VALIDATING_CONTENT_LENGTH) # Converting data to socket s = CDataToSocket(data) # Parsing socket to HTTP Response response = HTTPResponse(s) # Fetching content-length response.begin() content_length = response.getheader('Content-Length') # If no conetent-length in header - blocking if (not content_length): return False # DLP language = None try: language = self.dataLeakPreventor.detectCode(data) return not language except: return False pass # Reading content content = response.read(bytearray(CHandlerHTTP.CONTENT_LENGTH_MAX)) return not (CValidatorFile.isOfficeFile(content) and \ (CHandlerHTTP.CONTENT_LENGTH_MAX < long(content_length)))
def _do_handshake(self, url, version=0, origin=None, cookie=None): """http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00""" fields = [ "Upgrade: WebSocket", "Connection: Upgrade", "Host: %s" % (self.host,), "Sec-WebSocket-Draft: 2", ] if origin is None: origin = self.host fields.append("Origin: %s" % (origin,)) if cookie is not None: fields.append(cookie.output(header="Cookie:")) # Authenticate if required if self.username is not None: fields.append("Authorization: Basic %s" % b64encode('%s:%s' % (self.username, self.password or ''))) # What idiot invented this algorithm for key construction? spaces_1 = random.randint(1, 12) spaces_2 = random.randint(1, 12) max_1 = min(2**32-1 / spaces_1, 2**32-1) max_2 = min(2**32-1 / spaces_2, 2**32-1) number_1 = random.randint(0, max_1) number_2 = random.randint(0, max_2) product_1 = number_1 * spaces_1 product_2 = number_2 * spaces_2 key_1 = list(str(product_1)) key_2 = list(str(product_2)) ran_range = ''.join(map(chr, range(0x21, 0x30) + range(0x3a, 0x7f))) for n in xrange(random.randint(1, 12)): key_1.insert(random.randint(0, len(key_1)), random.choice(ran_range)) for n in xrange(random.randint(1, 12)): key_2.insert(random.randint(0, len(key_1)), random.choice(ran_range)) for n in xrange(spaces_1): key_1.insert(random.randint(1, len(key_1) - 1), ' ') for n in xrange(spaces_2): key_2.insert(random.randint(1, len(key_1) - 1), ' ') key_1 = ''.join(key_1) key_2 = ''.join(key_2) fields += [ "Sec-WebSocket-Key1: %s" % (key_1,), "Sec-WebSocket-Key2: %s" % (key_2,), ] key_3 = urandom(8) fields.append("Content-Length: %d" % (len(key_3),)) random.shuffle(fields) urlParts = urlparse.urlparse(url) # Finish request request = "GET %s HTTP/1.1\r\n" \ "%s\r\n" \ "\r\n" \ "%s" \ % (urlParts.path, '\r\n'.join(fields), key_3) if _debug: print >> sys.stderr, '\x1B[D\x1B[31m%s\x1B[m' % (request,), self.socket.sendall(request) response = HTTPResponse(self.socket) response.begin() if _debug: print >> sys.stderr, '\x1B[D\x1B[34m%s' % ({9: 'HTTP/0.9', 10: 'HTTP/1.0', 11: 'HTTP/1.1'}[response.version],), response.status, response.reason print >> sys.stderr, '%s\x1B[m' % (response.msg,) if response.status != 101: self.socket.close() raise RuntimeError("WebSocket upgrade failed: %d %s" % (response.status, response.reason)) # Hack to override httplib's setting it to 0 for 101's response.length = None expected_headers = ('upgrade', 'connection', 'sec-websocket-origin', 'sec-websocket-location') for header in expected_headers: if header not in response.msg: raise RuntimeError, "Expected WebSocket header not present: %s" % (header,) if response.msg['upgrade'].lower().strip() != 'websocket': raise RuntimeError, "Upgraded to wrong protocol, WebSocket expected: %s" % (response.msg['upgrade'],) if response.msg['connection'].lower().strip() != 'upgrade': raise RuntimeError, "Bad Connection header: %s" % (response.msg['connection'],) if response.msg['sec-websocket-origin'].lower().strip() != origin: raise RuntimeError, "Wrong WebSocket origin: %s" % (response.msg['sec-websocket-origin'],) if response.msg['sec-websocket-location'].lower().strip() != urlparse.urlunparse((self.scheme, self.host, urlParts.path, urlParts.params, urlParts.query, urlParts.fragment)): raise RuntimeError, "Bad WebSocket location: %s" % (response.msg['sec-websocket-location'],) assert 0 <= number_1 <= 4294967295 assert 0 <= number_2 <= 4294967295 challenge = struct.pack('>LL8s', number_1, number_2, key_3) expected = md5(challenge).digest() assert len(expected) == 16 reply = response.read(len(expected))
req += self.rfile.read(int(self.headers['Content-Length'])) # Send it down the pipe! self._proxy_sock.sendall(self.mitm_request(req)) # Parse response h = HTTPResponse(self._proxy_sock) h.begin() # Get rid of the pesky header del h.msg['Transfer-Encoding'] # Time to relay the message across res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason) res += '%s\r\n' % h.msg res += h.read() # Let's close off the remote end h.close() self._proxy_sock.close() # Relay the message self.request.sendall(self.mitm_response(res)) def mitm_request(self, data): for p in self.server._req_plugins: data = p(self.server, self).do_request(data) return data def mitm_response(self, data): for p in self.server._res_plugins:
from httplib import HTTPResponse from StringIO import StringIO import helpers, induce http_response_str = helpers.slurparg() class FakeSocket(): def __init__(self, response_str): self._file = StringIO(response_str) def makefile(self, *args, **kwargs): return self._file with induce.grammar() as g: with induce.Tracer(http_response_str, g): source = FakeSocket(http_response_str) response = HTTPResponse(source) response.begin() print "status:", response.status print "single header:", response.getheader('Content-Type') print "content:", response.read(len(http_response_str)) # the len here will give a 'big enough' value to read the whole content
def _command(self, uri, method='GET', body='', **kwargs): """Makes an HTTP request through to an AirPlay server Args: uri(string): The URI to request method(string): The HTTP verb to use when requesting `uri`, defaults to GET body(string): If provided, will be sent witout alteration as the request body. Content-Length header will be set to len(`body`) **kwargs: If provided, Will be converted to a query string and appended to `uri` Returns: True: Request returned 200 OK, with no response body False: Request returned something other than 200 OK, with no response body Mixed: The body of the HTTP response """ # generate the request if len(kwargs): uri = uri + '?' + urlencode(kwargs) request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str( len(body)) + "\r\n\r\n" + body try: request = bytes(request, 'UTF-8') except TypeError: pass # send it self.control_socket.send(request) # parse our response result = self.control_socket.recv(self.RECV_SIZE) resp = HTTPResponse(FakeSocket(result)) resp.begin() # if our content length is zero, then return bool based on result code if int(resp.getheader('content-length', 0)) == 0: if resp.status == 200: return True else: return False # else, parse based on provided content-type # and return the response body content_type = resp.getheader('content-type') if content_type is None: raise RuntimeError('Response returned without a content type!') if content_type == 'text/parameters': body = resp.read() try: body = str(body, 'UTF-8') except TypeError: pass return email.message_from_string(body) if content_type == 'text/x-apple-plist+xml': return plist_loads(resp.read()) raise RuntimeError( 'Response received with unknown content-type: {0}'.format( content_type))
def _command(self, uri, method='GET', body='', **kwargs): """Makes an HTTP request through to an AirPlay server Args: uri(string): The URI to request method(string): The HTTP verb to use when requesting `uri`, defaults to GET body(string): If provided, will be sent witout alteration as the request body. Content-Length header will be set to len(`body`) **kwargs: If provided, Will be converted to a query string and appended to `uri` Returns: True: Request returned 200 OK, with no response body False: Request returned something other than 200 OK, with no response body Mixed: The body of the HTTP response """ # generate the request if len(kwargs): uri = uri + '?' + urlencode(kwargs) request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str(len(body)) + "\r\n\r\n" + body try: request = bytes(request, 'UTF-8') except TypeError: pass # send it self.control_socket.send(request) # parse our response result = self.control_socket.recv(self.RECV_SIZE) resp = HTTPResponse(FakeSocket(result)) resp.begin() # if our content length is zero, then return bool based on result code if int(resp.getheader('content-length', 0)) == 0: if resp.status == 200: return True else: return False # else, parse based on provided content-type # and return the response body content_type = resp.getheader('content-type') if content_type is None: raise RuntimeError('Response returned without a content type!') if content_type == 'text/parameters': body = resp.read() try: body = str(body, 'UTF-8') except TypeError: pass return email.message_from_string(body) if content_type == 'text/x-apple-plist+xml': return plist_loads(resp.read()) raise RuntimeError('Response received with unknown content-type: {0}'.format(content_type))
def proxy(self): doInject = False inWhileList = False if gOptions.log > 0: print self.requestline port = 80 host = self.headers["Host"] if host.find(":") != -1: port = int(host.split(":")[1]) host = host.split(":")[0] try: redirectUrl = self.path while True: (scm, netloc, path, params, query, _) = urlparse.urlparse(redirectUrl) if gOptions.log > 2: print urlparse.urlparse(redirectUrl) if (netloc not in gConfig["REDIRECT_DOMAINS"]): break prefixes = gConfig["REDIRECT_DOMAINS"][netloc].split('|') found = False for prefix in prefixes: prefix = prefix + "=" for param in query.split('&') : if param.find(prefix) == 0: print "redirect to " + urllib.unquote(param[len(prefix):]) redirectUrl = urllib.unquote(param[len(prefix):]) found = True continue if not found: break if (host in gConfig["HSTS_DOMAINS"]): redirectUrl = "https://" + self.path[7:] #redirect if (redirectUrl != self.path): status = "HTTP/1.1 302 Found" self.wfile.write(status + "\r\n") self.wfile.write("Location: " + redirectUrl + "\r\n") return # Remove http://[host] , for google.com.hk path = self.path[self.path.find(netloc) + len(netloc):] connectHost = self.getip(host) if (host in gConfig["BLOCKED_DOMAINS"]) or isIpBlocked(connectHost): gConfig["BLOCKED_DOMAINS"][host] = True if gOptions.log>0 : print "add ip "+ connectHost + " to block list" return self.do_METHOD_Tunnel() if True: for d in domainWhiteList: if host.endswith(d): if gOptions.log > 0: print host + " in domainWhiteList: " + d inWhileList = True if not inWhileList: doInject = self.enableInjection(host, connectHost) self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if gOptions.log > 1: print "connect to " + host + ":" + str(port) self.remote.connect((connectHost, port)) if doInject: if gOptions.log > 0: print "inject http for "+host self.remote.send("\r\n\r\n") # Send requestline if path == "": path = "/" print " ".join((self.command, path, self.request_version)) + "\r\n" self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n") # Send headers if host[-12:] == ".appspot.com": print "add version code " + gConfig["VERSION"] + " in HTTP header" self.headers["X-WCProxy"] = gConfig["VERSION"] self.headers["X-WCPasswd"] = gConfig["PROXY_PASSWD"] self.remote.send(str(self.headers) + "\r\n") # Send Post data if(self.command=='POST'): self.remote.send(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) badStatusLine = False msg = "http405" try : response.begin() print host + " response: %d"%(response.status) msg = "http%d"%(response.status) except BadStatusLine: print host + " response: BadStatusLine" msg = "badStatusLine" badStatusLine = True except: raise if doInject and (response.status == 400 or response.status == 405 or badStatusLine): self.remote.close() self.remote = None if gOptions.log > 0: print host + " seem not support inject, " + msg domainWhiteList.append(host) return self.do_METHOD_Tunnel() # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status + "\r\n") h = '' for hh, vv in response.getheaders(): if hh.upper()!='TRANSFER-ENCODING': h += hh + ': ' + vv + '\r\n' self.wfile.write(h + "\r\n") dataLength = 0 while True: response_data = response.read(8192) if(len(response_data) == 0): break if dataLength == 0 and (len(response_data) <= 501): if response_data.find("<title>400 Bad Request") != -1 or response_data.find("<title>501 Method Not Implemented") != -1: print host + " not supporting injection" domainWhiteList.append(host) response_data = gConfig["PAGE_RELOAD_HTML"] self.wfile.write(response_data) dataLength += len(response_data) if gOptions.log > 1: print "data length: %d"%dataLength except: if self.remote: self.remote.close() self.remote = None (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) status = "HTTP/1.1 302 Found" if host in gConfig["HSTS_ON_EXCEPTION_DOMAINS"]: redirectUrl = "https://" + self.path[7:] self.wfile.write(status + "\r\n") self.wfile.write("Location: " + redirectUrl + "\r\n") exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == socket.error: code, msg = str(exc_value).split('] ') code = code[1:].split(' ')[1] if code in ["32", "10053"]: #errno.EPIPE, 10053 is for Windows if gOptions.log > 0: print "Detected remote disconnect: " + host return if code in ["61"]: #server not support injection if doInject: print "try not inject " + host domainWhiteList.append(host) self.do_METHOD_Tunnel() return print "error in proxy: ", self.requestline print exc_type print str(exc_value) + " " + host if exc_type == socket.timeout or (exc_type == socket.error and code in ["60", "110", "10060"]): #timed out, 10060 is for Windows if not inWhileList: if gOptions.log > 0: print "add "+host+" to blocked domains" gConfig["BLOCKED_DOMAINS"][host] = True return self.do_METHOD_Tunnel()
return except Exception, e: self.send_error(500, str(e)) return request = '%s %s %s\r\n' % (self.command, self.path, self.request_version) request += '%s\r\n' % self.headers if 'Content-Length' in self.headers: request += self.rfile.read(int(self.headers['Content-Length'])) self._pipe_socket.sendall(request) http_response = HTTPResponse(self._pipe_socket) http_response.begin() response = '%s %s %s\r\n' % (self.request_version, http_response.status, http_response.reason) response += '%s\r\n' % http_response.msg response += http_response.read() http_response.close() self._pipe_socket.close() try: self.request.sendall(response) except Exception, e: print 'ERROR request relay: ',e def do_GET(self): self.do_RELAY() def do_POST(self): self.do_RELAY()
return # Formatear respuesta REQUEST req = '%s %s %s\r\n' % (self.command, self.path, self.request_version) # Agregar HEADER a Respuesta req += '%s\r\n' % self.headers # Concatenar mensaje y cuerto si existe en la respuesta if 'Content-Length' in self.headers: req += self.rfile.read(int(self.headers['Content-Length'])) # Enviar todo al MEN IN THE MIDDLE!! self._proxy_sock.sendall(self.mitm_request(req)) # parsear la resuesta a objeto HTTPRESPONSE http_response = HTTPResponse(self._proxy_sock) http_response.begin() content = http_response.read() # Eliminar del header 'Transfer-Encoding' del http_response.msg['Transfer-Encoding'] if 'content-encoding' in http_response.msg.keys(): if http_response.msg['Content-Encoding'] == 'gzip': compress_data = StringIO.StringIO(content) gzipper = gzip.GzipFile(fileobj=compress_data) del http_response.msg['Content-Encoding'] content = gzipper.read() # Reenviar mensaje obtenido por proxy response = '%s %s %s\r\n' % ( self.request_version, http_response.status, http_response.reason) response += '%s\r\n' % http_response.msg
def sogouProxy(self): if "Host" in self.headers: print('Connecting: ' + self.headers["Host"]) if self.headers["Host"].startswith('chrome_dcp_proxy_pac.cnbeining'): #Give a PAC file self.wfile.write("HTTP/1.1 200 OK".encode('ascii') + b'\r\n') hstr = '''Host: 127.0.0.1 function FindProxyForURL(url, host) { if (url.substring(0,5) == 'http:' && !isPlainHostName(host) && !shExpMatch(host, '*.local') && !isInNet(dnsResolve(host), '10.0.0.0', '255.0.0.0') && !isInNet(dnsResolve(host), '172.16.0.0', '255.240.0.0') && !isInNet(dnsResolve(host), '192.168.0.0', '255.255.0.0') && !isInNet(dnsResolve(host), '127.0.0.0', '255.255.255.0') ) return 'PROXY ''' + server_ip + ':' + str(server_port) + '''; DIRECT'; return 'DIRECT'; }''' self.wfile.write(hstr + b'\r\n') return if self.remote is None or self.lastHost != self.headers["Host"]: if PROXY_MODE == 'HTTPS': context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) context.verify_mode = ssl.CERT_REQUIRED context.check_hostname = True context.load_default_certs() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(RemoteTimeout) self.remote = context.wrap_socket(s, server_hostname='proxy.googlezip.net') self.remote.connect(('proxy.googlezip.net', 443)) else: #HTTP self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.remote.settimeout(RemoteTimeout) self.remote.connect(("compress.googlezip.net", 80)) self.remote.sendall(self.requestline.encode('ascii') + b"\r\n") # Add Verification Tags self.headers["Chrome-Proxy"] = get_google_header() headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n") self.remote.sendall(headerstr.encode('ascii') + b"\r\n") # Send Post data if self.command == 'POST': self.remote.sendall(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) response.begin() # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status.encode('ascii') + b'\r\n') hlist = [] for line in response.msg.headers: # Fixed multiple values of a same name if 'TRANSFER-ENCODING' not in line.upper(): hlist.append(line) self.wfile.write("".join(hlist) + b'\r\n') if self.command == "CONNECT": # NO HTTPS, as Chrome DCP does not allow HTTPS traffic return else: while True: response_data = response.read(BufferSize) if not response_data: break self.wfile.write(response_data)
def proxy(self): doProxy = False inWhileList = False logging.info (self.requestline) port = 80 host = self.headers["Host"] if host.find(":") != -1: port = int(host.split(":")[1]) host = host.split(":")[0] (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) if host in ["127.0.0.1", "localhost"]: basedir = os.path.dirname(__file__) htmlTemplate = os.path.join(basedir, "index.html") htmlFile = open(htmlTemplate) html = htmlFile.read() htmlFile.close() status = "HTTP/1.1 200 OK" if path == "/save": postData = self.rfile.read(int(self.headers['Content-Length'])) data = urlparse.parse_qs(postData) logging.info(str(data)) key = data["id"][0] value = data["value"][0] if key in gConfig: if type(gConfig[key]) == type(True): if value == "true": gConfig[key] = True if value == "false": gConfig[key] = False else: gConfig[key] = type(gConfig[key]) (value) self.wfile.write(status + "\r\n\r\n" + value) return if path == "/add": postData = self.rfile.read(int(self.headers['Content-Length'])) data = urlparse.parse_qs(postData) if "BLOCKED_DOMAINS" in data: domain = data["BLOCKED_DOMAINS"][0] if domain[:4] == "http": (scm, netloc, path, params, query, _) = urlparse.urlparse(domain) domain = netloc gConfig["BLOCKED_DOMAINS"][domain] = True self.wfile.write("HTTP/1.1 302 FOUND\r\n" + "Location: /\r\n\r\n" + domain) return #TODO: pac for key in gConfig: if type(gConfig[key]) in [str,int] : html = html.replace("{"+key+"}", str(gConfig[key])) else : html = html.replace("{" + key + "}", str(gConfig[key])) self.wfile.write(status + "\r\n\r\n" + html) return try: if (gConfig["ADSHOSTON"] and host in gConfig["ADSHOST"]): status = "HTTP/1.1 404 Not Found" self.wfile.write(status + "\r\n\r\n") return # Remove http://[host] , for google.com.hk path = self.path[self.path.find(netloc) + len(netloc):] connectHost = self.getip(host) logging.info ("Resolved " + host + " => " + connectHost) if isDomainBlocked(host) or isIpBlocked(connectHost): self.remote = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) logging.info("connect to " + host + ":" + str(port) + " var socks5 proxy") self.remote.connect((connectHost, port)) doProxy = True else: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) logging.debug( "connect to " + host + ":" + str(port)) self.remote.connect((connectHost, port)) # Send requestline if path == "": path = "/" print " ".join((self.command, path, self.request_version)) + "\r\n" self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n") self.remote.send(str(self.headers) + "\r\n") # Send Post data if(self.command=='POST'): self.remote.send(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) response.begin() print host + " response: %d"%(response.status) # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status + "\r\n") h = '' for hh, vv in response.getheaders(): if hh.upper()!='TRANSFER-ENCODING': h += hh + ': ' + vv + '\r\n' self.wfile.write(h + "\r\n") dataLength = 0 while True: response_data = response.read(8192) if(len(response_data) == 0): break if dataLength == 0 and (len(response_data) <= 501): if response_data.find("<title>400 Bad Request") != -1 or response_data.find("<title>501 Method Not Implemented") != -1: logging.error( host + " not supporting injection") domainWhiteList.append(host) response_data = gConfig["PAGE_RELOAD_HTML"] self.wfile.write(response_data) dataLength += len(response_data) logging.debug( "data length: %d"%dataLength) except: if self.remote: self.remote.close() self.remote = None (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) status = "HTTP/1.1 302 Found" exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == socket.error: code = exc_value[0] if code in [32, 10053, errno.EPIPE]: #errno.EPIPE, 10053 is for Windows logging.info ("Detected remote disconnect: " + host) return if code == errno.ECONNREFUSED: logging.info ("Detected ECONNREFUSED: " + host) return if code in [54, 10054]: #reset logging.info(host + ": reset from " + connectHost) if not doProxy: gConfig["BLOCKED_IPS"][connectHost] = True return self.proxy() print "error in proxy: ", self.requestline print exc_type print str(exc_value) + " " + host
# Send it down the pipe! self._proxy_sock.sendall(self.mitm_request(req)) # Parse response h = HTTPResponse(self._proxy_sock) h.begin() # Get rid of the pesky header del h.msg['Transfer-Encoding'] # Time to relay the message across res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason) res += '%s\r\n' % h.msg try: res += h.read(h.length) except IncompleteRead, e: print e.message # Let's close off the remote end h.close() self._proxy_sock.close() # Relay the message self.request.sendall(self.mitm_response(res)) def mitm_request(self, data): for p in self.server._req_plugins: data = p(self.server, self).do_request(data) return data
def proxy(self): doInject = False try: print self.requestline self.supportCrLfPrefix = True port = 80 host = self.headers["Host"] if host.find(":") != -1: port = int(host.split(":")[1]) host = host.split(":")[0] # Remove http://[host] path = self.path[self.path.find(host) + len(host) :] connectHost = self.getip(host) doInject = self.enableInjection(host, connectHost) if self.remote is None or self.lastHost != self.headers["Host"]: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.remote.connect((connectHost, port)) if doInject: self.remote.send("\r\n\r\n") self.lastHost = self.headers["Host"] while True: # Send requestline self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n") # Send headers self.remote.send(str(self.headers) + "\r\n") # Send Post data if self.command == "POST": self.remote.send(self.rfile.read(int(self.headers["Content-Length"]))) response = HTTPResponse(self.remote, method=self.command) response.begin() if response.status == 400 and self.supportCrLfPrefix == True: while response.read(8192): pass self.supportCrLfPrefix = False continue break # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status + "\r\n") h = "" for hh, vv in response.getheaders(): if hh.upper() != "TRANSFER-ENCODING": h += hh + ": " + vv + "\r\n" self.wfile.write(h + "\r\n") while True: response_data = response.read(8192) if len(response_data) == 0: break self.wfile.write(response_data) except: if self.remote: self.remote.close() self.remote = None exc_type, exc_value, exc_traceback = sys.exc_info() print "error in proxy: ", self.requestline print exc_type print exc_value traceback.print_tb(exc_traceback) (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) if scm.upper() != "HTTP": self.wfile.write("HTTP/1.1 500 Server Error " + scm.upper() + "\r\n") elif netloc == urlparse.urlparse(PROXY_SERVER)[1]: self.wfile.write("HTTP/1.1 500 Server Error, Cannot connect to proxy " + "\r\n") else: if doInject: status = "HTTP/1.1 302 Found" self.wfile.write(status + "\r\n") self.wfile.write("Location: " + PROXY_SERVER + self.path[7:] + "\r\n") else: print ("Not redirect " + self.path) self.wfile.write("HTTP/1.1 500 Server Error Unkown Error\r\n") self.connection.close()
def proxy(self): doProxy = False inWhileList = False logging.info (self.requestline) port = 80 host = self.headers["Host"] if host.find(":") != -1: port = int(host.split(":")[1]) host = host.split(":")[0] (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) if host in ["127.0.0.1", "localhost"]: basedir = os.path.dirname(__file__) htmlTemplate = os.path.join(basedir, "index.html") htmlFile = open(htmlTemplate) html = htmlFile.read() htmlFile.close() status = "HTTP/1.1 200 OK" if path == "/save": postData = self.rfile.read(int(self.headers['Content-Length'])) data = urlparse.parse_qs(postData) logging.info(str(data)) key = data["id"][0] value = data["value"][0] if key in gConfig: if type(gConfig[key]) == type(True): if value == "true": gConfig[key] = True if value == "false": gConfig[key] = False else: gConfig[key] = type(gConfig[key]) (value) self.wfile.write(status + "\r\n\r\n" + value) return if path == "/add": postData = self.rfile.read(int(self.headers['Content-Length'])) data = urlparse.parse_qs(postData) if "BLOCKED_DOMAINS" in data: domain = data["BLOCKED_DOMAINS"][0] if domain[:4] == "http": (scm, netloc, path, params, query, _) = urlparse.urlparse(domain) domain = netloc gConfig["BLOCKED_DOMAINS"][domain] = True self.wfile.write("HTTP/1.1 302 FOUND\r\n" + "Location: /\r\n\r\n" + domain) return #TODO: pac for key in gConfig: if type(gConfig[key]) in [str,int] : html = html.replace("{"+key+"}", str(gConfig[key])) else : html = html.replace("{" + key + "}", str(gConfig[key])) self.wfile.write(status + "\r\n\r\n" + html) return try: if (gConfig["ADSHOSTON"] and host in gConfig["ADSHOST"]): status = "HTTP/1.1 404 Not Found" self.wfile.write(status + "\r\n\r\n") return # Remove http://[host] , for google.com.hk path = self.path[self.path.find(netloc) + len(netloc):] connectHost = self.getip(host) logging.info ("Resolved " + host + " => " + connectHost) if isDomainBlocked(host) or isIpBlocked(connectHost): self.remote = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) logging.info("connect to " + host + ":" + str(port) + " var socks5 proxy") self.remote.connect((connectHost, port)) doProxy = True else: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) logging.debug( "connect to " + host + ":" + str(port)) self.remote.connect((connectHost, port)) # Send requestline if path == "": path = "/" print (" ".join((self.command, path, self.request_version)) + "\r\n") self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n") self.remote.send(str(self.headers) + "\r\n") # Send Post data if(self.command=='POST'): self.remote.send(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) response.begin() print (host + " response: %d"%(response.status)) # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status + "\r\n") h = '' for hh, vv in response.getheaders(): if hh.upper()!='TRANSFER-ENCODING': h += hh + ': ' + vv + '\r\n' self.wfile.write(h + "\r\n") dataLength = 0 while True: response_data = response.read(8192) if(len(response_data) == 0): break if dataLength == 0 and (len(response_data) <= 501): if response_data.find("<title>400 Bad Request") != -1 or response_data.find("<title>501 Method Not Implemented") != -1: logging.error( host + " not supporting injection") domainWhiteList.append(host) response_data = gConfig["PAGE_RELOAD_HTML"] self.wfile.write(response_data) dataLength += len(response_data) logging.debug( "data length: %d"%dataLength) except: if self.remote: self.remote.close() self.remote = None (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) status = "HTTP/1.1 302 Found" exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == socket.error: code = exc_value[0] if code == errno.EPIPE: #errno.EPIPE, 10053 is for Windows logging.info ("Detected remote disconnect: " + host) return if code == errno.ECONNREFUSED: logging.info ("Detected ECONNREFUSED: " + host) return if code in errno.ECONNRESET: #reset logging.info(host + ": reset from " + connectHost) if not doProxy: gConfig["BLOCKED_IPS"][connectHost] = True return self.proxy() print ("error in proxy: ", self.requestline) print (exc_type) print (str(exc_value) + " " + host)
def mitm_response(self, data): lines = data.split("\r\n") r = HTTPResponse(FakeSocket(data)) r.begin() # response line self.doc.response.status = r.status self.doc.response.responseline = lines[0] # headers ct = "" cookies = list() for header in r.getheaders(): name = header[0] value = header[1] self.doc.add_parsed_response_header(name, value) if name == "content-type": ct = value elif name == "set-cookie": cookies.append(value) # content type try: m = reContentType.search(ct) self.doc.response.content_type = m.group(1) except: pass # cookies for cookie in cookies: # TODO: the following code extracts only partial cookie data - check/rewrite try: pc = SimpleCookie(cookie) for name in pc.keys(): c = pc[name] try: value = c.value except AttributeError: value = None try: domain = c.domain except AttributeError: domain = None try: path = c.path except AttributeError: path = None try: exp = c.expires except AttributeError: exp = None self.doc.add_response_cookie(name, value, domain, path, exp) except: pass # body bodybytes = r.read() self.doc.response.body = bodybytes.decode(args.charset, args.encodingerrors) try: self.doc.save(storeResponseBody) except: pass return data
def do_GET(self): if self.path == 'http://weevely/': self.send_cacert() return req = self content_length = int(req.headers.get('Content-Length', 0)) req_body = self.rfile.read(content_length) if content_length else '' if req.path[0] == '/': if isinstance(self.connection, ssl.SSLSocket): req.path = "https://%s%s" % (req.headers['Host'], req.path) else: req.path = "http://%s%s" % (req.headers['Host'], req.path) req.headers['Content-length'] = str(len(req_body)) u = urlparse.urlsplit(req.path) scheme, netloc, path = u.scheme, u.netloc, (u.path + '?' + u.query if u.query else u.path) assert scheme in ('http', 'https') if netloc: req.headers['Host'] = netloc setattr(req, 'headers', self.filter_headers(req.headers)) net_curl_args = [ '-X', self.command, '-i' ] net_curl_args.append(self.path) for h in req.headers: if h.title().lower() == 'host': host = self.headers[h] net_curl_args += [ '-H', '%s: %s' % ( h.title(), self.headers[h] ) ] if self.command == 'POST': content_len = int(self.headers.getheader('content-length', 0)) net_curl_args += [ '-d', req_body ] lock.acquire() try: result, headers, saved = ModuleExec( 'net_curl', net_curl_args ).run() finally: lock.release() if not headers: log.debug('Error no headers') self.send_error(502) return log.debug('> ' + '\r\n> '.join([ '%s: %s' % (h.title(), self.headers[h]) for h in self.headers ])) log.debug('< ' + '\r\n< '.join(headers)) http_response_str = '\r\n'.join(headers) + '\r\n\r\n' + result source = FakeSocket(http_response_str) res = HTTPResponse(source) res.begin() version_table = {10: 'HTTP/1.0', 11: 'HTTP/1.1'} setattr(res, 'headers', res.msg) setattr(res, 'response_version', version_table[res.version]) # support streaming if not 'Content-Length' in res.headers and 'no-store' in res.headers.get('Cache-Control', ''): setattr(res, 'headers', self.filter_headers(res.headers)) self.relay_streaming(res) return try: res_body = res.read() except Exception as e: log.debug(e) self.send_error(500) return setattr(res, 'headers', self.filter_headers(res.headers)) self.wfile.write("%s %d %s\r\n" % (self.protocol_version, res.status, res.reason)) for line in res.headers.headers: self.wfile.write(line) self.end_headers() self.wfile.write(res_body) self.wfile.flush()
def _command(self, uri, method='GET', body='', **kwargs): """Makes an HTTP request through to an AirPlay server Args: uri(string): The URI to request method(string): The HTTP verb to use when requesting `uri`, defaults to GET body(string): If provided, will be sent witout alteration as the request body. Content-Length header will be set to len(`body`) **kwargs: If provided, Will be converted to a query string and appended to `uri` Returns: True: Request returned 200 OK, with no response body False: Request returned something other than 200 OK, with no response body Mixed: The body of the HTTP response """ # generate the request if len(kwargs): uri = uri + '?' + urlencode(kwargs) request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str(len(body)) + "\r\n" request+="Host: %s:%s\r\n" % (self.host,self.port) request+="User-Agent: MediaControl/1.0\r\n" request+="X-Apple-Session-ID: c6c0033e-96f9-11e6-b0a4-a45e60c9debb\r\n" request+="Connection: close\r\n" request+="\r\n" request+=body try: if self.airplaySocket is None: self.airplaySocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.airplaySocket.settimeout(self.timeout) self.airplaySocket.connect((self.host, self.port)) except socket.error as exc: self.airplaySocket=None raise ValueError("Unable to connect to {0}:{1}: {2}".format(self.host, self.port, exc)) # send it rs=self.airplaySocket.sendall(request) # parse our response result = self.airplaySocket.recv(self.RECV_SIZE) resp = HTTPResponse(FakeSocket(result)) resp.begin() # if our content length is zero, then return bool based on result code if int(resp.getheader('content-length', 0)) == 0: if resp.status == 200: return True else: return False # else, parse based on provided content-type # and return the response body content_type = resp.getheader('content-type') if content_type is None: raise RuntimeError('Response returned without a content type!') return resp.read()
if 'WARC-Target-URI' in record: lengths.append(len(record['WARC-Target-URI'])) content = record.payload.read() src = FakeSocket(content) response = HTTPResponse(src) response.begin() content_type = response.getheader('Content-Type') if content_type is not None and "text/html" not in content_type: print "Skipped content", response.getheader('Content-Type') continue #content_lengths.append(len(content)) try: content = response.read() content_lengths.append(len(content)) h = BeautifulSoup(content, "lxml") except IncompleteRead: continue for title in h.findAll('title'): print "TITLE", title.text.encode('utf-8') for body in h.findAll("body"): #text = [i for i in body.recursiveChildGenerator() if type(i) == NavigableString] #text = [t.split() for t in text if len(t) > 0] #print(text) [s.extract() for s in body('script')] [s.extract() for s in body('style')]
# Append message body if present to the request if 'Content-Length' in self.headers: req += self.rfile.read(int(self.headers['Content-Length'])) # Send it down the pipe! self._proxy_sock.sendall(self.mitm_request(req)) # Parse response h = HTTPResponse(self._proxy_sock) h.begin() # Get rid of the pesky header del h.msg['Transfer-Encoding'] content = active_content_log(h.read(), h.msg, self.headers['Host']) # Time to relay the message across res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason) res += '%s\r\n' % h.msg res += content # Let's close off the remote end h.close() self._proxy_sock.close() # Relay the message self.request.sendall(self.mitm_response(res)) def mitm_request(self, data): for p in self.server._req_plugins:
def proxy(self): doInject = False try: print self.requestline port = 80 host = self.headers["Host"] if host.find(":") != -1: port = int(host.split(":")[1]) host = host.split(":")[0] redirectUrl = self.path while True: (scm, netloc, path, params, query, _) = urlparse.urlparse(redirectUrl) if (netloc not in gConfig["REDIRECT_DOMAINS"]): break prefixes = gConfig["REDIRECT_DOMAINS"][netloc].split('|') found = False for prefix in prefixes: prefix = prefix + "=" for param in query.split('&') : if param.find(prefix) == 0: print "redirect to " + urllib.unquote(param[len(prefix):]) redirectUrl = urllib.unquote(param[len(prefix):]) found = True continue if not found: break #redirect if (redirectUrl != self.path): status = "HTTP/1.1 302 Found" self.wfile.write(status + "\r\n") self.wfile.write("Location: " + redirectUrl + "\r\n") self.connection.close() return # Remove http://[host] path = self.path[self.path.find(netloc) + len(netloc):] connectHost = self.getip(host) self.lastHost = self.headers["Host"] while True: doInject = self.enableInjection(host, connectHost) if self.remote is None or self.lastHost != self.headers["Host"]: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) print "connect to " + host + ":" + str(port) self.remote.connect((connectHost, port)) if doInject: self.remote.send("\r\n\r\n") # Send requestline self.remote.send(" ".join((self.command, (doInject and [path] or [self.path])[0], self.request_version)) + "\r\n") # Send headers self.remote.send(str(self.headers) + "\r\n") # Send Post data if(self.command=='POST'): self.remote.send(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) badStatusLine = False msg = "http405" try : response.begin() print host + " response: %d"%(response.status) msg = "http%d"%(response.status) except BadStatusLine: print host + " response: BadStatusLine" msg = "badStatusLine" badStatusLine = True if doInject and (response.status == 400 or response.status == 405 or badStatusLine): self.remote.close() self.remote = None domainWhiteList.append(host) self.netlog(msg + "/host/" + host) continue break # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status + "\r\n") h = '' for hh, vv in response.getheaders(): if hh.upper()!='TRANSFER-ENCODING': h += hh + ': ' + vv + '\r\n' self.wfile.write(h + "\r\n") while True: response_data = response.read(8192) if(len(response_data) == 0): break self.wfile.write(response_data) except: if self.remote: self.remote.close() self.remote = None exc_type, exc_value, exc_traceback = sys.exc_info() print "error in proxy: ", self.requestline print exc_type print str(exc_value) + " " + host path = "" if exc_type == socket.error: code, msg = str(exc_value).split('] ') code = code[1:].replace(" ", "") path = code + "/host/" + host + "/?msg=" + urllib.quote(msg) traceback.print_tb(exc_traceback) (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) status = "HTTP/1.1 302 Found" if (netloc != urlparse.urlparse( gConfig["PROXY_SERVER"] )[1]): self.wfile.write(status + "\r\n") self.wfile.write("Location: " + gConfig["PROXY_SERVER"] + self.path[7:] + "\r\n") else: status = "HTTP/1.1 302 Found" if (scm.upper() != "HTTP"): msg = "schme-not-supported" else: msg = "web-proxy-fail" path = ("error/host/" + host + "/?msg=" + msg) self.wfile.write(status + "\r\n") self.wfile.write("Location: http://liruqi.info/post/18486575704/west-chamber-proxy#" + msg + "\r\n") self.netlog(path) self.connection.close()
def proxy(self): inWhileList = False logging.info (self.requestline) port = 80 host = self.headers["Host"] if host.find(":") != -1: port = int(host.split(":")[1]) host = host.split(":")[0] (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) if host in ["127.0.0.1", "localhost"]: basedir = os.path.dirname(__file__) htmlTemplate = os.path.join(basedir, "index.html") htmlFile = open(htmlTemplate) html = htmlFile.read() htmlFile.close() status = "HTTP/1.1 200 OK" if path == "/save": postData = self.rfile.read(int(self.headers['Content-Length'])) data = urlparse.parse_qs(postData) logging.info(str(data)) key = data["id"][0] value = data["value"][0] if key in gConfig: if type(gConfig[key]) == type(True): if value == "true": gConfig[key] = True if value == "false": gConfig[key] = False else: gConfig[key] = type(gConfig[key]) (value) self.wfile.write(status + "\r\n\r\n" + value) return if path == "/add": postData = self.rfile.read(int(self.headers['Content-Length'])) data = urlparse.parse_qs(postData) if "BLOCKED_DOMAINS" in data: domain = data["BLOCKED_DOMAINS"][0] if domain[:4] == "http": (scm, netloc, path, params, query, _) = urlparse.urlparse(domain) domain = netloc gConfig["BLOCKED_DOMAINS"][domain] = True self.wfile.write("HTTP/1.1 302 FOUND\r\n" + "Location: /\r\n\r\n" + domain) return for key in gConfig: if type(gConfig[key]) in [str,int] : html = html.replace("{"+key+"}", str(gConfig[key])) else : html = html.replace("{" + key + "}", str(gConfig[key])) self.wfile.write(status + "\r\n\r\n" + html) return try: if (host in gConfig["HSTS_DOMAINS"]): redirectUrl = "https://" + self.path[7:] #redirect status = "HTTP/1.1 302 Found" self.wfile.write(status + "\r\n") self.wfile.write("Location: " + redirectUrl + "\r\n\r\n") return for d in domainWhiteList: if host.endswith(d): logging.info (host + " in domainWhiteList: " + d) inWhileList = True connectHost = self.getip(host) # Remove http://[host] , for google.com.hk path = self.path[self.path.find(netloc) + len(netloc):] if path == "": path = "/" if isDomainBlocked(host) or isIpBlocked(connectHost): if gConfig['PROXY_TYPE'] == 'socks5': self.remote = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) else: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) connectHost = gConfig['HTTP_PROXY'] port = gConfig['HTTP_PROXY_PORT'] path = self.path else: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) logging.debug( host + ":connect to " + connectHost + ":" + str(port)) self.remote.connect((connectHost, port)) print " ".join((self.command, self.path, self.request_version)) + "\r\n" self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n" + str(self.headers) + "\r\n") # Send Post data if(self.command=='POST'): self.remote.send(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) badStatusLine = False try : response.begin() print host + " response: %d"%(response.status) except: self.remote.close() self.remote = None raise # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status + "\r\n") h = '' for hh, vv in response.getheaders(): if hh.upper()!='TRANSFER-ENCODING': h += hh + ': ' + vv + '\r\n' self.wfile.write(h + "\r\n") dataLength = 0 while True: response_data = response.read(8192) if(len(response_data) == 0): break self.wfile.write(response_data) dataLength += len(response_data) logging.debug( "data length: %d"%dataLength) self.remote.close() self.remote = None except: if self.remote: self.remote.close() self.remote = None (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) status = "HTTP/1.1 302 Found" if host in gConfig["HSTS_ON_EXCEPTION_DOMAINS"]: redirectUrl = "https://" + self.path[7:] self.wfile.write(status + "\r\n") self.wfile.write("Location: " + redirectUrl + "\r\n") exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == socket.error: code = exc_value[0] if code in [32, 10053]: #errno.EPIPE, 10053 is for Windows logging.info ("Detected remote disconnect: " + host) return if code in [54, 10054]: #reset logging.info(host + ": reset from " + connectHost) gConfig["BLOCKED_IPS"][connectHost] = True return print "error in proxy: ", self.requestline, exc_type print str(exc_value) + " " + host if exc_type == socket.timeout or (exc_type == socket.error and code in [60, 110, 10060]): #timed out, 10060 is for Windows if not inWhileList: logging.info ("add "+host+" to blocked domains") gConfig["BLOCKED_IPS"][connectHost] = True
def proxy(self): doInject = False inWhileList = False logging.info(self.requestline) port = 80 host = self.headers["Host"] if host.find(":") != -1: port = int(host.split(":")[1]) host = host.split(":")[0] (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) if host in ["127.0.0.1", "localhost"]: basedir = os.path.dirname(__file__) htmlTemplate = os.path.join(basedir, "index.html") htmlFile = open(htmlTemplate) html = htmlFile.read() htmlFile.close() status = "HTTP/1.1 200 OK" if path == "/save": postData = self.rfile.read(int(self.headers['Content-Length'])) data = urlparse.parse_qs(postData) logging.info(str(data)) key = data["id"][0] value = data["value"][0] if key in gConfig: if type(gConfig[key]) == type(True): if value == "true": gConfig[key] = True if value == "false": gConfig[key] = False else: gConfig[key] = type(gConfig[key])(value) hookInit() self.wfile.write(status + "\r\n\r\n" + value) return if path == "/add": postData = self.rfile.read(int(self.headers['Content-Length'])) data = urlparse.parse_qs(postData) if "BLOCKED_DOMAINS" in data: domain = data["BLOCKED_DOMAINS"][0] if domain[:4] == "http": (scm, netloc, path, params, query, _) = urlparse.urlparse(domain) domain = netloc gConfig["BLOCKED_DOMAINS"][domain] = True self.wfile.write("HTTP/1.1 302 FOUND\r\n" + "Location: /\r\n\r\n" + domain) return for key in gConfig: if type(gConfig[key]) in [str, int]: html = html.replace("{" + key + "}", str(gConfig[key])) else: html = html.replace("{" + key + "}", str(gConfig[key])) self.wfile.write(status + "\r\n\r\n" + html) return try: if (host in gConfig["HSTS_DOMAINS"]): redirectUrl = "https://" + self.path[7:] #redirect status = "HTTP/1.1 302 Found" self.wfile.write(status + "\r\n") self.wfile.write("Location: " + redirectUrl + "\r\n\r\n") return if (gConfig["ADSHOSTON"] and host in gConfig["ADSHOST"]): status = "HTTP/1.1 404 Not Found" self.wfile.write(status + "\r\n\r\n") return # Remove http://[host] , for google.com.hk path = self.path[self.path.find(netloc) + len(netloc):] for d in domainWhiteList: if host.endswith(d): logging.info(host + " in domainWhiteList: " + d) inWhileList = True connectHost = self.getip(host) if not inWhileList: doInject = self.enableInjection(host, connectHost) logging.info("Resolved " + host + " => " + connectHost) if isDomainBlocked(host) or isIpBlocked(connectHost): if gConfig["PROXY_TYPE"] == "socks5": self.remote = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) logging.info("connect to " + host + ":" + str(port) + " var socks5 proxy") self.remote.connect((connectHost, port)) else: logging.info(host + " blocked, try goagent.") return self.do_METHOD_Tunnel() else: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) logging.debug("connect to " + host + ":" + str(port)) self.remote.connect((connectHost, port)) if doInject: logging.info("inject http for " + host) self.remote.send("\r\n\r\n") # Send requestline if path == "": path = "/" print " ".join((self.command, path, self.request_version)) + "\r\n" self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n") self.remote.send(str(self.headers) + "\r\n") # Send Post data if (self.command == 'POST'): self.remote.send( self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) badStatusLine = False msg = "http405" try: response.begin() print host + " response: %d" % (response.status) msg = "http%d" % (response.status) except BadStatusLine: print host + " response: BadStatusLine" msg = "badStatusLine" badStatusLine = True except: raise if doInject and (response.status == 400 or response.status == 405 or badStatusLine): self.remote.close() self.remote = None logging.info(host + " seem not support inject, " + msg) return self.do_METHOD_Tunnel() # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status + "\r\n") h = '' for hh, vv in response.getheaders(): if hh.upper() != 'TRANSFER-ENCODING': h += hh + ': ' + vv + '\r\n' self.wfile.write(h + "\r\n") dataLength = 0 while True: response_data = response.read(8192) if (len(response_data) == 0): break if dataLength == 0 and (len(response_data) <= 501): if response_data.find( "<title>400 Bad Request" ) != -1 or response_data.find( "<title>501 Method Not Implemented") != -1: logging.error(host + " not supporting injection") domainWhiteList.append(host) response_data = gConfig["PAGE_RELOAD_HTML"] self.wfile.write(response_data) dataLength += len(response_data) logging.debug("data length: %d" % dataLength) except: if self.remote: self.remote.close() self.remote = None (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) status = "HTTP/1.1 302 Found" if host in gConfig["HSTS_ON_EXCEPTION_DOMAINS"]: redirectUrl = "https://" + self.path[7:] self.wfile.write(status + "\r\n") self.wfile.write("Location: " + redirectUrl + "\r\n") exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == socket.error: code, msg = str(exc_value).split('] ') code = code[1:].split(' ')[1] if code in ["32", "10053"]: #errno.EPIPE, 10053 is for Windows logging.info("Detected remote disconnect: " + host) return if code in ["54"]: return self.do_METHOD_Tunnel() if code in ["61"]: #server not support injection if doInject: logging.info("try not inject " + host) domainWhiteList.append(host) return self.do_METHOD_Tunnel() print "error in proxy: ", self.requestline print exc_type print str(exc_value) + " " + host if exc_type == socket.timeout or ( exc_type == socket.error and code in ["60", "110", "10060" ]): #timed out, 10060 is for Windows if not inWhileList: logging.info("add " + host + " to blocked domains") gConfig["BLOCKED_DOMAINS"][host] = True return self.do_METHOD_Tunnel()
def proxy(self): doInject = False if gOptions.log > 0: print self.requestline port = 80 host = self.headers["Host"] if host.find(":") != -1: port = int(host.split(":")[1]) host = host.split(":")[0] errpath = "" try: redirectUrl = self.path while True: (scm, netloc, path, params, query, _) = urlparse.urlparse(redirectUrl) if gOptions.log > 2: print urlparse.urlparse(redirectUrl) if (netloc not in gConfig["REDIRECT_DOMAINS"]): break prefixes = gConfig["REDIRECT_DOMAINS"][netloc].split('|') found = False for prefix in prefixes: prefix = prefix + "=" for param in query.split('&') : if param.find(prefix) == 0: print "redirect to " + urllib.unquote(param[len(prefix):]) redirectUrl = urllib.unquote(param[len(prefix):]) found = True continue if not found: break if (host in gConfig["HSTS_DOMAINS"]): redirectUrl = "https://" + self.path[7:] #redirect if (redirectUrl != self.path): status = "HTTP/1.1 302 Found" self.wfile.write(status + "\r\n") self.wfile.write("Location: " + redirectUrl + "\r\n") self.wfile.close() return # Remove http://[host] , for google.com.hk path = self.path[self.path.find(netloc) + len(netloc):] if host in gConfig["BLOCKED_DOMAINS"]: host = gConfig["PROXY_SERVER_SIMPLE"] path = self.path[len(scm)+2:] self.headers["Host"] = gConfig["PROXY_SERVER_SIMPLE"] print "use simple web proxy for " + path self.lastHost = self.headers["Host"] while True: inWhileList = False for d in domainWhiteList: if host.endswith(d): if gOptions.log > 1: print host + " in domainWhiteList: " + d inWhileList = True connectHost = host if not inWhileList: connectHost = self.getip(host) if connectHost in gConfig["BLOCKED_IPS"]: gConfig["BLOCKED_DOMAINS"][host] = True host = gConfig["PROXY_SERVER_SIMPLE"] path = self.path[len(scm)+2:] self.headers["Host"] = gConfig["PROXY_SERVER_SIMPLE"] connectHost = self.getip(host) print "use simple web proxy for " + path doInject = self.enableInjection(host, connectHost) if self.remote is None or self.lastHost != self.headers["Host"]: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if gOptions.log > 1: print "connect to " + host + ":" + str(port) self.remote.connect((connectHost, port)) if doInject: if gOptions.log > 0: print "inject http for "+host self.remote.send("\r\n\r\n") # Send requestline if path == "": path = "/" print " ".join((self.command, path, self.request_version)) + "\r\n" self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n") # Send headers self.remote.send(str(self.headers) + "\r\n") # Send Post data if(self.command=='POST'): self.remote.send(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) badStatusLine = False msg = "http405" try : response.begin() print host + " response: %d"%(response.status) msg = "http%d"%(response.status) except BadStatusLine: print host + " response: BadStatusLine" msg = "badStatusLine" badStatusLine = True except: raise if doInject and (response.status == 400 or response.status == 405 or badStatusLine) and host != gConfig["PROXY_SERVER_SIMPLE"]: self.remote.close() self.remote = None domainWhiteList.append(host) errpath = (msg + "/host/" + host) continue break # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status + "\r\n") h = '' for hh, vv in response.getheaders(): if hh.upper()!='TRANSFER-ENCODING': h += hh + ': ' + vv + '\r\n' self.wfile.write(h + "\r\n") dataLength = 0 while True: response_data = response.read(8192) if(len(response_data) == 0): break if dataLength == 0 and (len(response_data) <= 320): if response_data.find("<title>400 Bad Request") != -1 or response_data.find("<title>501 Method Not Implemented") != -1: print host + " not supporting injection" domainWhiteList.append(host) response_data = gConfig["PAGE_RELOAD_HTML"] self.wfile.write(response_data) dataLength += len(response_data) if gOptions.log > 1: print "data length: %d"%dataLength self.wfile.close() except: if self.remote: self.remote.close() self.remote = None exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == socket.error: code, msg = str(exc_value).split('] ') code = code[1:].split(' ')[1] if code in ["32", "10053"]: #errno.EPIPE, 10053 is for Windows if gOptions.log > 0: print "Detected remote disconnect: " + host self.wfile.close() return if code in ["61"]: #server not support injection if doInject: print "try not inject " + host domainWhiteList.append(host) self.proxy() return print "error in proxy: ", self.requestline print exc_type print str(exc_value) + " " + host errpath = "unkown/host/" + host if exc_type == socket.timeout or (exc_type == socket.error and code in ["60", "110", "10060"]): #timed out, 10060 is for Windows if gOptions.log > 0: print "add "+host+" to blocked domains" gConfig["BLOCKED_DOMAINS"][host] = True self.wfile.write("HTTP/1.1 200 OK\r\n\r\n") self.wfile.write(gConfig["PAGE_RELOAD_HTML"]) return traceback.print_tb(exc_traceback) (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) status = "HTTP/1.1 302 Found" if (netloc != urlparse.urlparse( gConfig["PROXY_SERVER"] )[1] and doInject): self.wfile.write(status + "\r\n") redirectUrl = gConfig["PROXY_SERVER"] + self.path[7:] if host in gConfig["HSTS_ON_EXCEPTION_DOMAINS"]: redirectUrl = "https://" + self.path[7:] self.wfile.write("Location: " + redirectUrl + "\r\n") else: if (scm.upper() != "HTTP"): msg = "schme-not-supported" else: msg = "web-proxy-fail" errpath = ("error/host/" + host + "/?msg=" + msg) self.wfile.write(status + "\r\n") self.wfile.write("Location: http://westchamberproxy.appspot.com/#" + msg + "\r\n") self.wfile.close() print "client connection closed" if errpath != "": self.netlog(errpath)
def do_COMMAND(self): global allCache global dumpToDisk cacheHit = False sendNotModifiesResponse = False cacheurl = self.path print cacheurl # checking if we have a cache hit if not ('?' in cacheurl) and (cacheurl in allCache): #in all cache array add two more fields max-age and date,time when added, and check if the object is valid before #making cachehit true cacheHit = True #------ code below uncomment # if 'If-Modified-Since' in self.headers: # last_modified_date = self.headers['If-Modified-Since'] # formatter_string = "%a, %d %b %Y %H:%M:%S %Z" # in_req_modified_date = datetime.strptime(last_modified_date, formatter_string) # #get the other date from cache # in_cache_modified_date = allCache[cacheurl][3] # if in_req_modified_date > in_cache_modified_date: # #can be served from cache sendNotModifiesResponse = False # No cache hit, fetch from Internet if not cacheHit: # Is this an SSL tunnel? if not self.is_connect: try: # Connect to destination self._connect_to_host() except Exception, e: self.send_error(500, str(e)) return # Extract path # Build request req = '%s %s %s\r\n' % (self.command, self.path, self.request_version) # Add headers to the request req += '%s\r\n' % self.headers # Append message body if present to the request if 'Content-Length' in self.headers: req += self.rfile.read(int(self.headers['Content-Length'])) # Send it down the pipe! self._proxy_sock.sendall(self.mitm_request(req)) # Parse response h = HTTPResponse(self._proxy_sock) h.begin() # Get rid of the pesky header del h.msg['Transfer-Encoding'] # Time to relay the message across res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason) res += '%s\r\n' % h.msg toWrite=h.read() res += toWrite # fetched an object from the Internet, need to add it to memory and dump it to disk if not ("?" in cacheurl): fileName=cacheFileName() tempObject=CacheObject(cacheurl, h.getheaders(), h.status, h.reason, toWrite,fileName) # current_datetime = datetime.now() # allCache[cacheurl]=['memory', tempObject, fileName, current_datetime] dumpToDisk.append(tempObject) h.close() self._proxy_sock.close()
if 'Content-Length' in self.headers: req += self.rfile.read(int(self.headers['Content-Length'])) try: # Send it down the pipe! self._proxy_sock.sendall(self.mitm_request(req)) # Parse response h = HTTPResponse(self._proxy_sock) h.begin() # Get rid of the pesky header del h.msg['Transfer-Encoding'] # Time to relay the message across res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason) res += '%s\r\n' % h.msg content_received = h.read() res += content_received #log the size try: http_obj = recorder.http_object(h.getheaders(), url_requested, content_received, h.status, h.reason, self.request_version, webpage, self.rtt) recorder.save_obj(http_obj) except Exception, e: print str(e) self.request.sendall(self.mitm_response(res)) except SocketError as e: if e.errno != errno.ECONNRESET:
def mitm_response(self, data): lines = data.split("\r\n") r = HTTPResponse(FakeSocket(data)) r.begin() # response line self.doc.response.status = r.status self.doc.response.responseline = lines[0].decode(args.charset, args.encodingerrors) # headers ct = "" cookies = list() for header in r.getheaders(): name = header[0].decode(args.charset, args.encodingerrors) value = header[1].decode(args.charset, args.encodingerrors) self.doc.add_parsed_response_header(name, value) if name == "content-type": ct = value elif name == "set-cookie": cookies.append(value) # content type try: m = reContentType.search(ct) self.doc.response.content_type = m.group(1) except: pass # cookies for cookie in cookies: # TODO: the following code extracts only partial cookie data - check/rewrite try: pc = SimpleCookie(cookie) for name in pc.keys(): c = pc[name] try: value = c.value except AttributeError: value = None try: domain = c.domain except AttributeError: domain = None try: path = c.path except AttributeError: path = None try: exp = c.expires except AttributeError: exp = None self.doc.add_response_cookie(name, value, domain, path, exp) except: pass # body bodybytes = r.read() self.doc.response.body = bodybytes.decode(args.charset, args.encodingerrors) self.doc.save(storeResponseBody) return data
def proxy(self): doInject = False inWhileList = False logging.info (self.requestline) port = 80 host = self.headers["Host"] if host.find(":") != -1: port = int(host.split(":")[1]) host = host.split(":")[0] (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) if host in ["127.0.0.1", "localhost"]: basedir = os.path.dirname(__file__) htmlTemplate = os.path.join(basedir, "index.html") htmlFile = open(htmlTemplate) html = htmlFile.read() htmlFile.close() status = "HTTP/1.1 200 OK" if path == "/save": postData = self.rfile.read(int(self.headers['Content-Length'])) data = urlparse.parse_qs(postData) logging.info(str(data)) key = data["id"][0] value = data["value"][0] if key in gConfig: if type(gConfig[key]) == type(True): if value == "true": gConfig[key] = True if value == "false": gConfig[key] = False else: gConfig[key] = type(gConfig[key]) (value) hookInit() self.wfile.write(status + "\r\n\r\n" + value) return for key in gConfig: if type(gConfig[key]) in [str,int] : html = html.replace("{"+key+"}", str(gConfig[key])) else : html = html.replace("{" + key + "}", str(gConfig[key])) self.wfile.write(status + "\r\n\r\n" + html) return try: if (host in gConfig["HSTS_DOMAINS"]): redirectUrl = "https://" + self.path[7:] #redirect status = "HTTP/1.1 302 Found" self.wfile.write(status + "\r\n") self.wfile.write("Location: " + redirectUrl + "\r\n\r\n") return if (gConfig["ADSHOSTON"] and host in gConfig["ADSHOST"]): status = "HTTP/1.1 404 Not Found" self.wfile.write(status + "\r\n\r\n") return # Remove http://[host] , for google.com.hk path = self.path[self.path.find(netloc) + len(netloc):] connectHost = self.getip(host) logging.info ("Resolved " + host + " => " + connectHost) rootDomain = string.join(host.split('.')[-2:], '.') if True: for d in domainWhiteList: if host.endswith(d): logging.info (host + " in domainWhiteList: " + d) inWhileList = True if not inWhileList: doInject = self.enableInjection(host, connectHost) if isDomainBlocked(host) or isIpBlocked(connectHost): if gConfig["PROXY_TYPE"] == "socks5": self.remote = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) logging.info("connect to " + host + ":" + str(port) + " var socks5 proxy") self.remote.connect((connectHost, port)) else: logging.info(host + " blocked, try goagent.") return self.do_METHOD_Tunnel() else: self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) logging.debug( "connect to " + host + ":" + str(port)) self.remote.connect((connectHost, port)) if doInject: logging.info ("inject http for "+host) self.remote.send("\r\n\r\n") # Send requestline if path == "": path = "/" print " ".join((self.command, path, self.request_version)) + "\r\n" self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n") self.remote.send(str(self.headers) + "\r\n") # Send Post data if(self.command=='POST'): self.remote.send(self.rfile.read(int(self.headers['Content-Length']))) response = HTTPResponse(self.remote, method=self.command) badStatusLine = False msg = "http405" try : response.begin() print host + " response: %d"%(response.status) msg = "http%d"%(response.status) except BadStatusLine: print host + " response: BadStatusLine" msg = "badStatusLine" badStatusLine = True except: raise if doInject and (response.status == 400 or response.status == 405 or badStatusLine): self.remote.close() self.remote = None logging.info (host + " seem not support inject, " + msg) return self.do_METHOD_Tunnel() # Reply to the browser status = "HTTP/1.1 " + str(response.status) + " " + response.reason self.wfile.write(status + "\r\n") h = '' for hh, vv in response.getheaders(): if hh.upper()!='TRANSFER-ENCODING': h += hh + ': ' + vv + '\r\n' self.wfile.write(h + "\r\n") dataLength = 0 while True: response_data = response.read(8192) if(len(response_data) == 0): break if dataLength == 0 and (len(response_data) <= 501): if response_data.find("<title>400 Bad Request") != -1 or response_data.find("<title>501 Method Not Implemented") != -1: logging.error( host + " not supporting injection") domainWhiteList.append(host) response_data = gConfig["PAGE_RELOAD_HTML"] self.wfile.write(response_data) dataLength += len(response_data) logging.debug( "data length: %d"%dataLength) except: if self.remote: self.remote.close() self.remote = None (scm, netloc, path, params, query, _) = urlparse.urlparse(self.path) status = "HTTP/1.1 302 Found" if host in gConfig["HSTS_ON_EXCEPTION_DOMAINS"]: redirectUrl = "https://" + self.path[7:] self.wfile.write(status + "\r\n") self.wfile.write("Location: " + redirectUrl + "\r\n") exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type == socket.error: code, msg = str(exc_value).split('] ') code = code[1:].split(' ')[1] if code in ["32", "10053"]: #errno.EPIPE, 10053 is for Windows logging.info ("Detected remote disconnect: " + host) return if code in ["61"]: #server not support injection if doInject: logging.info( "try not inject " + host) domainWhiteList.append(host) self.do_METHOD_Tunnel() return print "error in proxy: ", self.requestline print exc_type print str(exc_value) + " " + host if exc_type == socket.timeout or (exc_type == socket.error and code in ["60", "110", "10060"]): #timed out, 10060 is for Windows if not inWhileList: logging.info ("add "+host+" to blocked domains") gConfig["BLOCKED_DOMAINS"][host] = True return self.do_METHOD_Tunnel()
try: # Send it down the pipe! self._proxy_sock.sendall(self.mitm_request(req)) # Parse response h = HTTPResponse(self._proxy_sock) h.begin() # Get rid of the pesky header del h.msg['Transfer-Encoding'] # Time to relay the message across res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason) res += '%s\r\n' % h.msg content_received = h.read() res += content_received HTTPObject_received = controller.HTTPObject(h.getheaders(), url_requested , content_received, h.status) controller.createObject(HTTPObject_received) # Relay the message self.request.sendall(self.mitm_response(res)) except SocketError as e: if e.errno != errno.ECONNRESET: raise pass # Let's close off the remote end if h != None: