def process_http(raw):
headers, body = raw.split(BODY_SEP, 1)
if headers.startswith('CONNECT'):
return None
if not len(body):
return {
'headers': headers,
'body': None
}
source = FakeSocket(raw)
response = HTTPResponse(source)
response.begin()
body = None
if response.getheader('Content-Encoding') == 'gzip':
buf = StringIO(response.read(len(raw)))
f = gzip.GzipFile(fileobj=buf)
body = f.read()
else:
body = response.read(len(raw))
return {
'headers': headers,
'body': body if len(body) else None
}
def setUp(self):
from httplib import HTTPResponse
with Stub() as HTTPResponse:
HTTPResponse.status >> 200
HTTPResponse.read() >> '{"id": "THE-PRECIOUS-GUID", "title": "example title", "subtitle": "example title - subtitle", "description": "a description", "user": "******", "tags": [], "created_at": 1329767353.0, "source": "http://www.example.com/example.csv", "link": "http://www.junar.com/datastreams/some-url"}'
with Stub() as conn:
from httplib import HTTPConnection
conn = HTTPConnection("api.junar.com")
conn.request("POST", "/datastreams/publish", self.params, self.headers)
conn.getresponse() >> HTTPResponse
def get_json(self, response):
res_str = str(helpers.bytesToString(response.getResponse()))
source = FakeSocket(res_str)
res_obj = HTTPResponse(source)
res_obj.begin()
print "Reading response object"
return json.loads(res_obj.read())
def send_request(self, request):
self.request_num += 1
# Sanity check: if we're sending a request with a content-length but
# we don't have that many bytes to send, we'll just get a 504. Don't
# send it and instead report a client error.
parts = request.split('\r\n\r\n', 1)
if len(parts) > 1:
req, body = parts
match = content_length_re.search(req)
if match:
if len(body) < int(match.group(1)):
self.error("request body of incorrect size")
return True
if not self.connection:
self._connect()
if self.connection:
# tally request method
#self.tally(request.split(" ", 1)[0])
try:
self.connection.sendall(request)
response = HTTPResponse(self.connection)
response.begin()
self.tally(response.status)
while response.read():
pass
if response.will_close:
# We hope our Connection: keep-alive won't be ignored, but
# deal with it if it does.
self._disconnect()
if self.options.speedup < 0.8:
# if we're slowing down by too much, keep-alive will just
# result in the server getting bored between requests and
# dropping the connection, so disable it.
self._disconnect()
return True
except IncompleteRead:
self.error("error while reading response: IncompleteRead (terminating job)")
self._disconnect()
except Exception, e: # TODO: more restrictive error catching?
self.error("error while sending request and reading response: %s %s" % (type(e), e))
self._disconnect()
if self.connection:
self.connection.close()
self.connection = None
def sendAndReceive(self):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((ORIGINAL_IP, 80))
sock.sendall(str(self))
except:
return GlobalTools.FailedHTTPResponse(), ''
res = HTTPResponse(sock)
res.begin()
res_body = res.read()
res.close()
if 'transfer-encoding' in res.msg:
# httplib.HTTPResponse automatically concatenate chunked response
# but do not delete 'transfer-encoding' header
# so the header must be deleted
res.msg.__delitem__('transfer-encoding')
compmeth = res.msg.getheader('content-encoding', '').lower()
if compmeth and compmeth.find('identity') != 0:
# response body is compressed with some method
offset = 0
if compmeth.find('gzip') != -1:
# if body is gziped, header offset value is 47
# if not, offset value is 0
# this server does not support sdch...
offset += 47
res_body = decompress(res_body, offset)
res.msg['content-encoding'] = 'identity'
return res, res_body
def send_and_recv(self):
try:
# because www.dream-pro.info is tlanslated to 127.0.0.1 using hosts' entry,
# send message to www.dream-pro.info with socket.socket to make
# http connection
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((WWW_DREAM_PRO_INFO,80))
sock.sendall(str(self))
except:
print 'SocketError'
return
res = HTTPResponse(sock)
res.begin()
res_body = res.read()
res.close()
if 'transfer-encoding' in res.msg:
# httplib.HTTPResponse automatically concatenate chunked response
# but do not delete 'transfer-encoding' header
# so the header must be deleted
res.msg.__delitem__('transfer-encoding')
compmeth = res.msg.getheader('content-encoding','').lower()
if compmeth and compmeth.find('identity') != 0 :
# response body is compressed with some method
offset = 0
if compmeth.find('gzip') != -1:
# if body is gziped, header offset value is 47
# if not, offset value is 0
# this server does not support sdch...
offset += 47
res_body = decompress(res_body,offset)
res.msg['content-encoding'] = 'identity'
return res, res_body
def process_record(self, record):
content = None
try:
payload = record.payload.read()
s = FakeSocket(payload)
response = HTTPResponse(s)
response.begin()
status_code = response.status
if status_code != 200:
return
content_type = response.getheader('Content-Type', '')
if 'text/html' not in content_type:
return
headers = response.getheaders()
content = response.read(len(payload))
except Exception:
self.increment_counter('errors', 'process_record', 1)
logging.error('Error processing record: {}', traceback.format_exc())
return
if content is not None:
content = content.strip()
if not content:
return
for item in self.process_content(record.url, headers, content):
yield item
def sogouProxy(self):
if self.remote is None or self.lastHost != self.headers["Host"]:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.settimeout(RemoteTimeout)
self.remote.connect((proxy_host, proxy_port))
self.remote.sendall(self.requestline.encode('ascii').replace('baike.baidu.com', '220.181.111.247') + b"\r\n")
# Add Sogou Verification Tags
self.headers["X-Sogou-Auth"] = x_sogou_auth
t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
self.headers["X-Sogou-Tag"] = calc_sogou_hash(t, self.headers['Host'])
self.headers["X-Sogou-Timestamp"] = t
headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
# Send Post data
if self.command == 'POST':
self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
hlist = []
for line in response.msg.headers: # Fixed multiple values of a same name
if 'TRANSFER-ENCODING' not in line.upper():
hlist.append(line)
self.wfile.write("".join(hlist) + b'\r\n')
if self.command == "CONNECT" and response.status == 200:
return self.transfer(self.remote, self.connection)
else:
while True:
response_data = response.read(BufferSize)
if not response_data: break
self.wfile.write(response_data)
def testPipelining(self):
# Tests the use of several requests issued at once.
s = ("GET / HTTP/1.0\r\n"
"Connection: %s\r\n"
"Content-Length: %d\r\n"
"\r\n"
"%s")
to_send = ''
count = 25
for n in range(count):
body = "Response #%d\r\n" % (n + 1)
if n + 1 < count:
conn = 'keep-alive'
else:
conn = 'close'
to_send += s % (conn, len(body), body)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(to_send)
for n in range(count):
expect_body = "Response #%d\r\n" % (n + 1)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(response_body))
self.failUnlessEqual(response_body, expect_body)
def proxy(self):
if self.remote is None or self.lastHost != self.headers['Host']:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.connect((proxy_host, proxy_port))
self.remote.sendall(self.requestline.encode('ascii') + b'\r\n')
# Add auth inf.
self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(auth)
#self.remote.sendall('Proxy-Authorization: Basic %s\r\n' % base64.b64encode(auth))
headerstr = str(self.headers).replace('\r\n', '\n').replace('\n', '\r\n')
self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
# Send Post data
if self.command == 'POST':
self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
# Reply to the browser
status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
hlist = []
for line in response.msg.headers: # Fixed multiple values of a same name
if 'TRANSFER-ENCODING' not in line.upper():
hlist.append(line)
self.wfile.write(''.join(hlist) + b'\r\n')
if self.command == 'CONNECT' and response.status == 200:
return self.transfer(self.remote, self.connection)
else:
while True:
response_data = response.read(BufferSize)
if not response_data: break
self.wfile.write(response_data)
def sogouProxy(self):
if self.remote is None or self.lastHost != self.headers["Host"]:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.settimeout(RemoteTimeout)
self.remote.connect((proxy_host, proxy_port))
self.remote.sendall(self.requestline.encode('ascii') + b"\r\n")
# Add Sogou Verification Tags
self.headers["X-Sogou-Auth"] = x_sogou_auth
t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
self.headers["X-Sogou-Tag"] = calc_sogou_hash(t, self.headers['Host'])
self.headers["X-Sogou-Timestamp"] = t
headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
# Send Post data
if self.command == 'POST':
self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
hlist = []
for line in response.msg.headers: # Fixed multiple values of a same name
if 'TRANSFER-ENCODING' not in line.upper():
hlist.append(line)
self.wfile.write("".join(hlist) + b'\r\n')
if self.command == "CONNECT" and response.status == 200:
return self.transfer(self.remote, self.connection)
else:
while True:
response_data = response.read(BufferSize)
if not response_data: break
self.wfile.write(response_data)
def testPipelining(self):
# Tests the use of several requests issued at once.
s = ("GET / HTTP/1.0\r\n"
"Connection: %s\r\n"
"Content-Length: %d\r\n"
"\r\n"
"%s")
to_send = ''
count = 25
for n in range(count):
body = "Response #%d\r\n" % (n + 1)
if n + 1 < count:
conn = 'keep-alive'
else:
conn = 'close'
to_send += s % (conn, len(body), body)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(to_send)
for n in range(count):
expect_body = "Response #%d\r\n" % (n + 1)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(response_body))
self.failUnlessEqual(response_body, expect_body)
def sogouProxy(self):
if self.headers["Host"].startswith('chrome_dcp_proxy_pac.cnbeining'): #Give a PAC file
self.wfile.write("HTTP/1.1 200 OK".encode('ascii') + b'\r\n')
hstr = '''Host: 127.0.0.1
function FindProxyForURL(url, host) {
if (url.substring(0,5) == 'http:' &&
!isPlainHostName(host) &&
!shExpMatch(host, '*.local') &&
!isInNet(dnsResolve(host), '10.0.0.0', '255.0.0.0') &&
!isInNet(dnsResolve(host), '172.16.0.0', '255.240.0.0') &&
!isInNet(dnsResolve(host), '192.168.0.0', '255.255.0.0') &&
!isInNet(dnsResolve(host), '127.0.0.0', '255.255.255.0') )
return 'PROXY ''' + server_ip + ':' + str(server_port) + '''; DIRECT';
return 'DIRECT';
}'''
self.wfile.write(hstr + b'\r\n')
return
if self.remote is None or self.lastHost != self.headers["Host"]:
if PROXY_MODE == 'HTTPS':
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
context.verify_mode = ssl.CERT_REQUIRED
context.check_hostname = True
context.load_default_certs()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(RemoteTimeout)
self.remote = context.wrap_socket(s, server_hostname='proxy.googlezip.net')
self.remote.connect(('proxy.googlezip.net', 443))
else: #HTTP
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.settimeout(RemoteTimeout)
self.remote.connect(("compress.googlezip.net", 80))
self.remote.sendall(self.requestline.encode('ascii') + b"\r\n")
# Add Verification Tags
self.headers["Chrome-Proxy"] = get_google_header()
headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
# Send Post data
if self.command == 'POST':
self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
hlist = []
for line in response.msg.headers: # Fixed multiple values of a same name
if 'TRANSFER-ENCODING' not in line.upper():
hlist.append(line)
self.wfile.write("".join(hlist) + b'\r\n')
if self.command == "CONNECT": # NO HTTPS, as Chrome DCP does not allow HTTPS traffic
return
else:
while True:
response_data = response.read(BufferSize)
if not response_data: break
self.wfile.write(response_data)
def _get_response(self):
# Parse response
h = HTTPResponse(self._proxy_sock)
h.begin()
res_d = {
'status':
h.status,
'reason':
h.reason,
'req_version':
self.request_version,
'msg':
h.msg,
'trans_enc': (h.msg['Transfer-Encoding']
if 'Transfer-Encoding' in h.msg else '[NO-ENCODING]')
}
# Get rid of the pesky header
del h.msg['Transfer-Encoding']
res = '%(req_version)s %(status)s %(reason)s\r\n%(msg)s\r\n' % res_d
res_d['data'] = h.read()
res_d['time'] = time()
res += res_d['data']
# Let's close off the remote end
h.close()
self._proxy_sock.close()
# Relay the message
self.request.sendall(self.mitm_response(res))
print res
return res_d
def send_and_recv(self):
try:
# because www.dream-pro.info is tlanslated to 127.0.0.1 using hosts' entry,
# send message to www.dream-pro.info with socket.socket to make
# http connection
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((WWW_DREAM_PRO_INFO, 80))
sock.sendall(str(self))
except:
print 'SocketError'
return
res = HTTPResponse(sock)
res.begin()
res_body = res.read()
res.close()
if 'transfer-encoding' in res.msg:
# httplib.HTTPResponse automatically concatenate chunked response
# but do not delete 'transfer-encoding' header
# so the header must be deleted
res.msg.__delitem__('transfer-encoding')
compmeth = res.msg.getheader('content-encoding', '').lower()
if compmeth and compmeth.find('identity') != 0:
# response body is compressed with some method
offset = 0
if compmeth.find('gzip') != -1:
# if body is gziped, header offset value is 47
# if not, offset value is 0
# this server does not support sdch...
offset += 47
res_body = decompress(res_body, offset)
res.msg['content-encoding'] = 'identity'
return res, res_body
def do_proxy(self):
try:
if self.s == 0:
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((proxy_host, proxy_port))
self.s.send(self.requestline.encode('ascii') + b'\r\n')
# Add Sogou Verification Tags
self.headers['X-Sogou-Auth'] = x_sogou_auth
t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
self.headers['X-Sogou-Tag'] = calc_sogou_hash(t, self.headers['Host'])
self.headers['X-Sogou-Timestamp'] = t
self.s.send(str(self.headers).encode('ascii') + b'\r\n')
# Send Post data
if(self.command=='POST'):
self.s.send(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.s, method=self.command, buffering=True)
response.begin()
# Reply to the browser
status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
h = ''
for hh, vv in response.getheaders():
if hh.upper()!='TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h.encode('ascii') + b'\r\n')
while True:
response_data = response.read(8192)
if(len(response_data) == 0):
break
self.wfile.write(response_data)
except socket.error:
print('socket error for ' + self.requestline)
def open_image2(url, buff_size=4096, connect_timeout=5, read_timeout=4,
full_read_timeout=30, logger=logging):
t = urlparse.urlparse(url)
path = t.path or '/'
if len(t.netloc.split(':')) == 2:
HOST, PORT = t.netloc.split(':')
else:
HOST, PORT = t.netloc, 80
UA = ('Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/54.0.2840.87 Safari/537.36')
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(connect_timeout)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
s.connect((HOST, PORT))
except Exception as e:
logger.exception(e)
return None
s.send("GET %s HTTP/1.1\r\n" % path)
s.send("Host:%s\r\n" % HOST)
s.send("Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n")
s.send("User-Agent:%s\r\n" % UA)
s.send('\r\n')
data = []
begin = time.time()
while 1:
if time.time() - begin > full_read_timeout:
logger.error('reach full_read_timeout')
data = []
break
try:
msg = s.recv(buff_size)
#print len(msg), len(data), time.time() - begin
if len(msg) == 0:
break
data.append(msg)
except socket.timeout as e:
if e.args[0] == 'timed out':
break
else:
logger.error('open image [%s] socket timeout: %s' % (url, str(e)))
data = []
break
except socket.error as e:
logger.error('open image [%s] socket error: %s' % (url, str(e)))
data = []
break
s.shutdown(1)
s.close()
data = ''.join(data)
if data:
resp = HTTPResponse(FakeSocket(data))
resp.begin()
if resp.status == 200:
return resp.read(len(data))
return None
def get(self, request):
# Format and pass along the initial request
request = self.format_request(request)
self.outsocket.sendall(request)
# Wait for request
while (True):
r, w, x = select.select([self.insocket, self.outsocket], [], [], 0)
if self.insocket in r:
request = self.insocket.recv(buff_size)
if request == "":
print "Got end message from browser"
self.kill()
else:
try:
request = self.format_request(request)
except MismatchedHost:
print "Host changed. Getting new end socket"
self.insocket.sendall("")
self.kill()
self.outsocket.sendall(request)
if self.outsocket in r:
httpRes = HTTPResponse(self.outsocket)
response = ""
try:
httpRes.begin()
headers = str(httpRes.msg)
print headers
content = httpRes.read()
# TODO: Move below to format_response
# Fix chunked header
if headers.find("Transfer-Encoding: chunked") != -1:
headers = headers.replace(
"Transfer-Encoding: chunked\r\n", "")
headers += "Content Length: " + str(
len(content)) + "\r\n"
if httpRes.version == 10:
response += "HTTP/1.0 "
elif httpRes.version == 11:
response += "HTTP/1.1 "
response += str(httpRes.status) + " " + str(
httpRes.reason) + nl
response += headers + nl
response += content
print response
except BadStatusLine:
self.kill()
self.insocket.sendall(response)
self.kill()
if self.killed:
self.end()
return
def filter_malware(response_str):
global apikey
source = HTTPObject(response_str)
http_response = HTTPResponse(source)
http_response.begin()
url = 'https://www.virustotal.com/vtapi/v2/file/report'
content = http_response.read(len(response_str))
hash = hashlib.md5(content).hexdigest()
params = {'apikey': apikey, 'resource': hash}
response = requests.get(url, params=params)
return response
def getBody(msg):
''' Get the entity body section of the http message '''
class FakeSocket():
def __init__(self, msg):
self.file = StringIO(msg)
def makefile(self, *args, **kwargs):
return self.file
source = FakeSocket(msg)
rsp = HTTPResponse(source)
rsp.begin()
return rsp.read(len(msg))
def decompress_payload(payload):
try:
source = FakeSocket(payload)
res = HTTPResponse(source)
res.begin()
result = zlib.decompress(res.read(), 16 + zlib.MAX_WBITS)
except Exception as e:
result = payload
# try:
# result = '.'.join(str(ord(c)) for c in payload)
# except:
# result = payload
return result
def get(self, request):
# Format and pass along the initial request
request = self.format_request(request)
self.outsocket.sendall(request)
# Wait for request
while(True):
r,w,x = select.select([self.insocket, self.outsocket],[],[],0)
if self.insocket in r:
request = self.insocket.recv(buff_size)
if request == "":
print "Got end message from browser"
self.kill()
else:
try:
request = self.format_request(request)
except MismatchedHost:
print "Host changed. Getting new end socket"
self.insocket.sendall("")
self.kill()
self.outsocket.sendall(request)
if self.outsocket in r:
httpRes = HTTPResponse(self.outsocket)
response = ""
try:
httpRes.begin()
headers = str(httpRes.msg)
content = httpRes.read()
if headers.find("Transfer-Encoding: chunked") != -1:
headers = headers.replace("Transfer-Encoding: chunked\r\n", "")
headers += "Content Length: " + str(len(content)) + "\r\n"
if httpRes.version == 10:
response += "HTTP/1.0 "
elif httpRes.version == 11:
response += "HTTP/1.1 "
response += str(httpRes.status) + " " + str(httpRes.reason) + nl
response += headers + nl
response += content
#print response
except BadStatusLine:
self.kill()
self.insocket.sendall(response)
self.kill()
#'''
if self.killed:
self.end()
return
def get(addr, path, debug=False):
client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
client.connect(addr)
client.send("GET {} HTTP/1.0\r\n\r\n".format(path))
resp_str = client.recv(65536)
source = FakeSocket(resp_str)
resp = HTTPResponse(source)
resp.begin()
if resp.status == 200:
text = resp.read(len(resp_str))
data = json.loads(text)
if debug:
log.debug(data)
return data
return {}
def get(addr, path, debug=False):
client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
client.connect(addr)
client.send("GET {} HTTP/1.0\r\n\r\n".format(path))
resp_str = client.recv(65536)
source = FakeSocket(resp_str)
resp = HTTPResponse(source)
resp.begin()
if resp.status == 200:
text = resp.read(len(resp_str))
data = json.loads(text)
if debug:
log.debug(data)
return data
return {}
def sendData(self, ip):
"""
套接字通信
:param ip:
:return:
"""
addr = (ip, 80)
data = "GET / HTTP/1.1\r\nHost: " + self.args.domain + "\r\n" + "Connection: close" + "\r\n\r\n"
recvdata = ""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
res = {}
res.setdefault("ip", ip)
res.setdefault("status")
res.setdefault("body")
res.setdefault("server_header")
res.setdefault("x_powered_by_header")
res.setdefault("success")
try:
sock.connect(addr)
sock.send(data)
while True:
buffer = sock.recv(1024)
if not buffer:
break
recvdata += buffer
response = HTTPResponse(FakeSocket(recvdata))
response.begin() # begin有什么用???
res["status"] = response.status
if response.status == 200: # 这里就可以保存为html文件了
msg = ip + " seems done!!!"
self.log.info(msg)
res["body"] = response.read()
res["server"] = response.getheader("Server", default="Known")
res["x_powered_by"] = response.getheader("X-Powered-By",
default="Known")
if self.args.keyword:
if self.args.keyword in res["body"]:
res["success"] = True
else:
res["success"] = False
else:
res["success"] = True
self.result.append(res)
except Exception as err:
self.log.error(err)
res["success"] = False
def get_json(self, response):
res_bytes = response.getResponse() # byte[]
if not res_bytes:
print "[!] Failed to get json for Response. Response has not bytes"
return {}
res_str = str(helpers.bytesToString(res_bytes))
source = FakeSocket(res_str)
res_obj = HTTPResponse(source)
res_obj.begin()
# print "Reading response object"
res_text = res_obj.read()
try:
res_data = json.loads(res_text)
except Exception as e:
print "[!] Failed to convert to json: " + str(res_text)
res_data = {}
return res_data
def testWithoutCRLF(self):
# Tests the use of just newlines rather than CR/LFs.
data = "Echo\nthis\r\nplease"
s = ("GET / HTTP/1.0\n"
"Connection: close\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(data))
self.failUnlessEqual(response_body, data)
def testWithoutCRLF(self):
# Tests the use of just newlines rather than CR/LFs.
data = "Echo\nthis\r\nplease"
s = ("GET / HTTP/1.0\n"
"Connection: close\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(data))
self.failUnlessEqual(response_body, data)
def isValidData(self, data):
print(MESSAGES.VALIDATING_CONTENT_LENGTH)
# Converting data to socket
s = CDataToSocket(data)
# Parsing socket to HTTP Response
response = HTTPResponse(s)
# Fetching content-length
response.begin()
content_length = response.getheader('Content-Length')
# If no conetent-length in header - blocking
if (not content_length):
return False
# DLP
language = None
try:
language = self.dataLeakPreventor.detectCode(data)
return not language
except:
return False
pass
# Reading content
content = response.read(bytearray(CHandlerHTTP.CONTENT_LENGTH_MAX))
return not (CValidatorFile.isOfficeFile(content) and \
(CHandlerHTTP.CONTENT_LENGTH_MAX < long(content_length)))
def _do_handshake(self, url, version=0, origin=None, cookie=None):
"""http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00"""
fields = [
"Upgrade: WebSocket",
"Connection: Upgrade",
"Host: %s" % (self.host,),
"Sec-WebSocket-Draft: 2",
]
if origin is None:
origin = self.host
fields.append("Origin: %s" % (origin,))
if cookie is not None:
fields.append(cookie.output(header="Cookie:"))
# Authenticate if required
if self.username is not None:
fields.append("Authorization: Basic %s"
% b64encode('%s:%s' % (self.username, self.password or '')))
# What idiot invented this algorithm for key construction?
spaces_1 = random.randint(1, 12)
spaces_2 = random.randint(1, 12)
max_1 = min(2**32-1 / spaces_1, 2**32-1)
max_2 = min(2**32-1 / spaces_2, 2**32-1)
number_1 = random.randint(0, max_1)
number_2 = random.randint(0, max_2)
product_1 = number_1 * spaces_1
product_2 = number_2 * spaces_2
key_1 = list(str(product_1))
key_2 = list(str(product_2))
ran_range = ''.join(map(chr, range(0x21, 0x30) + range(0x3a, 0x7f)))
for n in xrange(random.randint(1, 12)):
key_1.insert(random.randint(0, len(key_1)), random.choice(ran_range))
for n in xrange(random.randint(1, 12)):
key_2.insert(random.randint(0, len(key_1)), random.choice(ran_range))
for n in xrange(spaces_1):
key_1.insert(random.randint(1, len(key_1) - 1), ' ')
for n in xrange(spaces_2):
key_2.insert(random.randint(1, len(key_1) - 1), ' ')
key_1 = ''.join(key_1)
key_2 = ''.join(key_2)
fields += [
"Sec-WebSocket-Key1: %s" % (key_1,),
"Sec-WebSocket-Key2: %s" % (key_2,),
]
key_3 = urandom(8)
fields.append("Content-Length: %d" % (len(key_3),))
random.shuffle(fields)
urlParts = urlparse.urlparse(url)
# Finish request
request = "GET %s HTTP/1.1\r\n" \
"%s\r\n" \
"\r\n" \
"%s" \
% (urlParts.path, '\r\n'.join(fields), key_3)
if _debug:
print >> sys.stderr, '\x1B[D\x1B[31m%s\x1B[m' % (request,),
self.socket.sendall(request)
response = HTTPResponse(self.socket)
response.begin()
if _debug:
print >> sys.stderr, '\x1B[D\x1B[34m%s' % ({9: 'HTTP/0.9', 10: 'HTTP/1.0', 11: 'HTTP/1.1'}[response.version],), response.status, response.reason
print >> sys.stderr, '%s\x1B[m' % (response.msg,)
if response.status != 101:
self.socket.close()
raise RuntimeError("WebSocket upgrade failed: %d %s" % (response.status, response.reason))
# Hack to override httplib's setting it to 0 for 101's
response.length = None
expected_headers = ('upgrade', 'connection', 'sec-websocket-origin', 'sec-websocket-location')
for header in expected_headers:
if header not in response.msg:
raise RuntimeError, "Expected WebSocket header not present: %s" % (header,)
if response.msg['upgrade'].lower().strip() != 'websocket':
raise RuntimeError, "Upgraded to wrong protocol, WebSocket expected: %s" % (response.msg['upgrade'],)
if response.msg['connection'].lower().strip() != 'upgrade':
raise RuntimeError, "Bad Connection header: %s" % (response.msg['connection'],)
if response.msg['sec-websocket-origin'].lower().strip() != origin:
raise RuntimeError, "Wrong WebSocket origin: %s" % (response.msg['sec-websocket-origin'],)
if response.msg['sec-websocket-location'].lower().strip() != urlparse.urlunparse((self.scheme, self.host, urlParts.path, urlParts.params, urlParts.query, urlParts.fragment)):
raise RuntimeError, "Bad WebSocket location: %s" % (response.msg['sec-websocket-location'],)
assert 0 <= number_1 <= 4294967295
assert 0 <= number_2 <= 4294967295
challenge = struct.pack('>LL8s', number_1, number_2, key_3)
expected = md5(challenge).digest()
assert len(expected) == 16
reply = response.read(len(expected))
req += self.rfile.read(int(self.headers['Content-Length']))
# Send it down the pipe!
self._proxy_sock.sendall(self.mitm_request(req))
# Parse response
h = HTTPResponse(self._proxy_sock)
h.begin()
# Get rid of the pesky header
del h.msg['Transfer-Encoding']
# Time to relay the message across
res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason)
res += '%s\r\n' % h.msg
res += h.read()
# Let's close off the remote end
h.close()
self._proxy_sock.close()
# Relay the message
self.request.sendall(self.mitm_response(res))
def mitm_request(self, data):
for p in self.server._req_plugins:
data = p(self.server, self).do_request(data)
return data
def mitm_response(self, data):
for p in self.server._res_plugins:
from httplib import HTTPResponse
from StringIO import StringIO
import helpers, induce
http_response_str = helpers.slurparg()
class FakeSocket():
def __init__(self, response_str):
self._file = StringIO(response_str)
def makefile(self, *args, **kwargs):
return self._file
with induce.grammar() as g:
with induce.Tracer(http_response_str, g):
source = FakeSocket(http_response_str)
response = HTTPResponse(source)
response.begin()
print "status:", response.status
print "single header:", response.getheader('Content-Type')
print "content:", response.read(len(http_response_str)) # the len here will give a 'big enough' value to read the whole content
req += self.rfile.read(int(self.headers['Content-Length']))
# Send it down the pipe!
self._proxy_sock.sendall(self.mitm_request(req))
# Parse response
h = HTTPResponse(self._proxy_sock)
h.begin()
# Get rid of the pesky header
del h.msg['Transfer-Encoding']
# Time to relay the message across
res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason)
res += '%s\r\n' % h.msg
res += h.read()
# Let's close off the remote end
h.close()
self._proxy_sock.close()
# Relay the message
self.request.sendall(self.mitm_response(res))
def mitm_request(self, data):
for p in self.server._req_plugins:
data = p(self.server, self).do_request(data)
return data
def mitm_response(self, data):
for p in self.server._res_plugins:
def _command(self, uri, method='GET', body='', **kwargs):
"""Makes an HTTP request through to an AirPlay server
Args:
uri(string): The URI to request
method(string): The HTTP verb to use when requesting `uri`, defaults to GET
body(string): If provided, will be sent witout alteration as the request body.
Content-Length header will be set to len(`body`)
**kwargs: If provided, Will be converted to a query string and appended to `uri`
Returns:
True: Request returned 200 OK, with no response body
False: Request returned something other than 200 OK, with no response body
Mixed: The body of the HTTP response
"""
# generate the request
if len(kwargs):
uri = uri + '?' + urlencode(kwargs)
request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str(
len(body)) + "\r\n\r\n" + body
try:
request = bytes(request, 'UTF-8')
except TypeError:
pass
# send it
self.control_socket.send(request)
# parse our response
result = self.control_socket.recv(self.RECV_SIZE)
resp = HTTPResponse(FakeSocket(result))
resp.begin()
# if our content length is zero, then return bool based on result code
if int(resp.getheader('content-length', 0)) == 0:
if resp.status == 200:
return True
else:
return False
# else, parse based on provided content-type
# and return the response body
content_type = resp.getheader('content-type')
if content_type is None:
raise RuntimeError('Response returned without a content type!')
if content_type == 'text/parameters':
body = resp.read()
try:
body = str(body, 'UTF-8')
except TypeError:
pass
return email.message_from_string(body)
if content_type == 'text/x-apple-plist+xml':
return plist_loads(resp.read())
raise RuntimeError(
'Response received with unknown content-type: {0}'.format(
content_type))
def _command(self, uri, method='GET', body='', **kwargs):
"""Makes an HTTP request through to an AirPlay server
Args:
uri(string): The URI to request
method(string): The HTTP verb to use when requesting `uri`, defaults to GET
body(string): If provided, will be sent witout alteration as the request body.
Content-Length header will be set to len(`body`)
**kwargs: If provided, Will be converted to a query string and appended to `uri`
Returns:
True: Request returned 200 OK, with no response body
False: Request returned something other than 200 OK, with no response body
Mixed: The body of the HTTP response
"""
# generate the request
if len(kwargs):
uri = uri + '?' + urlencode(kwargs)
request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str(len(body)) + "\r\n\r\n" + body
try:
request = bytes(request, 'UTF-8')
except TypeError:
pass
# send it
self.control_socket.send(request)
# parse our response
result = self.control_socket.recv(self.RECV_SIZE)
resp = HTTPResponse(FakeSocket(result))
resp.begin()
# if our content length is zero, then return bool based on result code
if int(resp.getheader('content-length', 0)) == 0:
if resp.status == 200:
return True
else:
return False
# else, parse based on provided content-type
# and return the response body
content_type = resp.getheader('content-type')
if content_type is None:
raise RuntimeError('Response returned without a content type!')
if content_type == 'text/parameters':
body = resp.read()
try:
body = str(body, 'UTF-8')
except TypeError:
pass
return email.message_from_string(body)
if content_type == 'text/x-apple-plist+xml':
return plist_loads(resp.read())
raise RuntimeError('Response received with unknown content-type: {0}'.format(content_type))
def proxy(self):
doInject = False
inWhileList = False
if gOptions.log > 0: print self.requestline
port = 80
host = self.headers["Host"]
if host.find(":") != -1:
port = int(host.split(":")[1])
host = host.split(":")[0]
try:
redirectUrl = self.path
while True:
(scm, netloc, path, params, query, _) = urlparse.urlparse(redirectUrl)
if gOptions.log > 2: print urlparse.urlparse(redirectUrl)
if (netloc not in gConfig["REDIRECT_DOMAINS"]):
break
prefixes = gConfig["REDIRECT_DOMAINS"][netloc].split('|')
found = False
for prefix in prefixes:
prefix = prefix + "="
for param in query.split('&') :
if param.find(prefix) == 0:
print "redirect to " + urllib.unquote(param[len(prefix):])
redirectUrl = urllib.unquote(param[len(prefix):])
found = True
continue
if not found:
break
if (host in gConfig["HSTS_DOMAINS"]):
redirectUrl = "https://" + self.path[7:]
#redirect
if (redirectUrl != self.path):
status = "HTTP/1.1 302 Found"
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + redirectUrl + "\r\n")
return
# Remove http://[host] , for google.com.hk
path = self.path[self.path.find(netloc) + len(netloc):]
connectHost = self.getip(host)
if (host in gConfig["BLOCKED_DOMAINS"]) or isIpBlocked(connectHost):
gConfig["BLOCKED_DOMAINS"][host] = True
if gOptions.log>0 : print "add ip "+ connectHost + " to block list"
return self.do_METHOD_Tunnel()
if True:
for d in domainWhiteList:
if host.endswith(d):
if gOptions.log > 0: print host + " in domainWhiteList: " + d
inWhileList = True
if not inWhileList:
doInject = self.enableInjection(host, connectHost)
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if gOptions.log > 1: print "connect to " + host + ":" + str(port)
self.remote.connect((connectHost, port))
if doInject:
if gOptions.log > 0: print "inject http for "+host
self.remote.send("\r\n\r\n")
# Send requestline
if path == "":
path = "/"
print " ".join((self.command, path, self.request_version)) + "\r\n"
self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n")
# Send headers
if host[-12:] == ".appspot.com":
print "add version code " + gConfig["VERSION"] + " in HTTP header"
self.headers["X-WCProxy"] = gConfig["VERSION"]
self.headers["X-WCPasswd"] = gConfig["PROXY_PASSWD"]
self.remote.send(str(self.headers) + "\r\n")
# Send Post data
if(self.command=='POST'):
self.remote.send(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
badStatusLine = False
msg = "http405"
try :
response.begin()
print host + " response: %d"%(response.status)
msg = "http%d"%(response.status)
except BadStatusLine:
print host + " response: BadStatusLine"
msg = "badStatusLine"
badStatusLine = True
except:
raise
if doInject and (response.status == 400 or response.status == 405 or badStatusLine):
self.remote.close()
self.remote = None
if gOptions.log > 0: print host + " seem not support inject, " + msg
domainWhiteList.append(host)
return self.do_METHOD_Tunnel()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status + "\r\n")
h = ''
for hh, vv in response.getheaders():
if hh.upper()!='TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h + "\r\n")
dataLength = 0
while True:
response_data = response.read(8192)
if(len(response_data) == 0): break
if dataLength == 0 and (len(response_data) <= 501):
if response_data.find("<title>400 Bad Request") != -1 or response_data.find("<title>501 Method Not Implemented") != -1:
print host + " not supporting injection"
domainWhiteList.append(host)
response_data = gConfig["PAGE_RELOAD_HTML"]
self.wfile.write(response_data)
dataLength += len(response_data)
if gOptions.log > 1: print "data length: %d"%dataLength
except:
if self.remote:
self.remote.close()
self.remote = None
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
status = "HTTP/1.1 302 Found"
if host in gConfig["HSTS_ON_EXCEPTION_DOMAINS"]:
redirectUrl = "https://" + self.path[7:]
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + redirectUrl + "\r\n")
exc_type, exc_value, exc_traceback = sys.exc_info()
if exc_type == socket.error:
code, msg = str(exc_value).split('] ')
code = code[1:].split(' ')[1]
if code in ["32", "10053"]: #errno.EPIPE, 10053 is for Windows
if gOptions.log > 0: print "Detected remote disconnect: " + host
return
if code in ["61"]: #server not support injection
if doInject:
print "try not inject " + host
domainWhiteList.append(host)
self.do_METHOD_Tunnel()
return
print "error in proxy: ", self.requestline
print exc_type
print str(exc_value) + " " + host
if exc_type == socket.timeout or (exc_type == socket.error and code in ["60", "110", "10060"]): #timed out, 10060 is for Windows
if not inWhileList:
if gOptions.log > 0: print "add "+host+" to blocked domains"
gConfig["BLOCKED_DOMAINS"][host] = True
return self.do_METHOD_Tunnel()
return
except Exception, e:
self.send_error(500, str(e))
return
request = '%s %s %s\r\n' % (self.command, self.path, self.request_version)
request += '%s\r\n' % self.headers
if 'Content-Length' in self.headers: request += self.rfile.read(int(self.headers['Content-Length']))
self._pipe_socket.sendall(request)
http_response = HTTPResponse(self._pipe_socket)
http_response.begin()
response = '%s %s %s\r\n' % (self.request_version, http_response.status, http_response.reason)
response += '%s\r\n' % http_response.msg
response += http_response.read()
http_response.close()
self._pipe_socket.close()
try:
self.request.sendall(response)
except Exception, e:
print 'ERROR request relay: ',e
def do_GET(self):
self.do_RELAY()
def do_POST(self):
self.do_RELAY()
return
# Formatear respuesta REQUEST
req = '%s %s %s\r\n' % (self.command, self.path, self.request_version)
# Agregar HEADER a Respuesta
req += '%s\r\n' % self.headers
# Concatenar mensaje y cuerto si existe en la respuesta
if 'Content-Length' in self.headers:
req += self.rfile.read(int(self.headers['Content-Length']))
# Enviar todo al MEN IN THE MIDDLE!!
self._proxy_sock.sendall(self.mitm_request(req))
# parsear la resuesta a objeto HTTPRESPONSE
http_response = HTTPResponse(self._proxy_sock)
http_response.begin()
content = http_response.read()
# Eliminar del header 'Transfer-Encoding'
del http_response.msg['Transfer-Encoding']
if 'content-encoding' in http_response.msg.keys():
if http_response.msg['Content-Encoding'] == 'gzip':
compress_data = StringIO.StringIO(content)
gzipper = gzip.GzipFile(fileobj=compress_data)
del http_response.msg['Content-Encoding']
content = gzipper.read()
# Reenviar mensaje obtenido por proxy
response = '%s %s %s\r\n' % (
self.request_version, http_response.status, http_response.reason)
response += '%s\r\n' % http_response.msg
def sogouProxy(self):
if "Host" in self.headers:
print('Connecting: ' + self.headers["Host"])
if self.headers["Host"].startswith('chrome_dcp_proxy_pac.cnbeining'): #Give a PAC file
self.wfile.write("HTTP/1.1 200 OK".encode('ascii') + b'\r\n')
hstr = '''Host: 127.0.0.1
function FindProxyForURL(url, host) {
if (url.substring(0,5) == 'http:' &&
!isPlainHostName(host) &&
!shExpMatch(host, '*.local') &&
!isInNet(dnsResolve(host), '10.0.0.0', '255.0.0.0') &&
!isInNet(dnsResolve(host), '172.16.0.0', '255.240.0.0') &&
!isInNet(dnsResolve(host), '192.168.0.0', '255.255.0.0') &&
!isInNet(dnsResolve(host), '127.0.0.0', '255.255.255.0') )
return 'PROXY ''' + server_ip + ':' + str(server_port) + '''; DIRECT';
return 'DIRECT';
}'''
self.wfile.write(hstr + b'\r\n')
return
if self.remote is None or self.lastHost != self.headers["Host"]:
if PROXY_MODE == 'HTTPS':
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
context.verify_mode = ssl.CERT_REQUIRED
context.check_hostname = True
context.load_default_certs()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(RemoteTimeout)
self.remote = context.wrap_socket(s, server_hostname='proxy.googlezip.net')
self.remote.connect(('proxy.googlezip.net', 443))
else: #HTTP
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.settimeout(RemoteTimeout)
self.remote.connect(("compress.googlezip.net", 80))
self.remote.sendall(self.requestline.encode('ascii') + b"\r\n")
# Add Verification Tags
self.headers["Chrome-Proxy"] = get_google_header()
headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
# Send Post data
if self.command == 'POST':
self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
hlist = []
for line in response.msg.headers: # Fixed multiple values of a same name
if 'TRANSFER-ENCODING' not in line.upper():
hlist.append(line)
self.wfile.write("".join(hlist) + b'\r\n')
if self.command == "CONNECT": # NO HTTPS, as Chrome DCP does not allow HTTPS traffic
return
else:
while True:
response_data = response.read(BufferSize)
if not response_data: break
self.wfile.write(response_data)
def proxy(self):
doProxy = False
inWhileList = False
logging.info (self.requestline)
port = 80
host = self.headers["Host"]
if host.find(":") != -1:
port = int(host.split(":")[1])
host = host.split(":")[0]
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
if host in ["127.0.0.1", "localhost"]:
basedir = os.path.dirname(__file__)
htmlTemplate = os.path.join(basedir, "index.html")
htmlFile = open(htmlTemplate)
html = htmlFile.read()
htmlFile.close()
status = "HTTP/1.1 200 OK"
if path == "/save":
postData = self.rfile.read(int(self.headers['Content-Length']))
data = urlparse.parse_qs(postData)
logging.info(str(data))
key = data["id"][0]
value = data["value"][0]
if key in gConfig:
if type(gConfig[key]) == type(True):
if value == "true": gConfig[key] = True
if value == "false": gConfig[key] = False
else:
gConfig[key] = type(gConfig[key]) (value)
self.wfile.write(status + "\r\n\r\n" + value)
return
if path == "/add":
postData = self.rfile.read(int(self.headers['Content-Length']))
data = urlparse.parse_qs(postData)
if "BLOCKED_DOMAINS" in data:
domain = data["BLOCKED_DOMAINS"][0]
if domain[:4] == "http":
(scm, netloc, path, params, query, _) = urlparse.urlparse(domain)
domain = netloc
gConfig["BLOCKED_DOMAINS"][domain] = True
self.wfile.write("HTTP/1.1 302 FOUND\r\n" + "Location: /\r\n\r\n" + domain)
return
#TODO: pac
for key in gConfig:
if type(gConfig[key]) in [str,int] :
html = html.replace("{"+key+"}", str(gConfig[key]))
else :
html = html.replace("{" + key + "}", str(gConfig[key]))
self.wfile.write(status + "\r\n\r\n" + html)
return
try:
if (gConfig["ADSHOSTON"] and host in gConfig["ADSHOST"]):
status = "HTTP/1.1 404 Not Found"
self.wfile.write(status + "\r\n\r\n")
return
# Remove http://[host] , for google.com.hk
path = self.path[self.path.find(netloc) + len(netloc):]
connectHost = self.getip(host)
logging.info ("Resolved " + host + " => " + connectHost)
if isDomainBlocked(host) or isIpBlocked(connectHost):
self.remote = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM)
logging.info("connect to " + host + ":" + str(port) + " var socks5 proxy")
self.remote.connect((connectHost, port))
doProxy = True
else:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logging.debug( "connect to " + host + ":" + str(port))
self.remote.connect((connectHost, port))
# Send requestline
if path == "":
path = "/"
print " ".join((self.command, path, self.request_version)) + "\r\n"
self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n")
self.remote.send(str(self.headers) + "\r\n")
# Send Post data
if(self.command=='POST'):
self.remote.send(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
print host + " response: %d"%(response.status)
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status + "\r\n")
h = ''
for hh, vv in response.getheaders():
if hh.upper()!='TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h + "\r\n")
dataLength = 0
while True:
response_data = response.read(8192)
if(len(response_data) == 0): break
if dataLength == 0 and (len(response_data) <= 501):
if response_data.find("<title>400 Bad Request") != -1 or response_data.find("<title>501 Method Not Implemented") != -1:
logging.error( host + " not supporting injection")
domainWhiteList.append(host)
response_data = gConfig["PAGE_RELOAD_HTML"]
self.wfile.write(response_data)
dataLength += len(response_data)
logging.debug( "data length: %d"%dataLength)
except:
if self.remote:
self.remote.close()
self.remote = None
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
status = "HTTP/1.1 302 Found"
exc_type, exc_value, exc_traceback = sys.exc_info()
if exc_type == socket.error:
code = exc_value[0]
if code in [32, 10053, errno.EPIPE]: #errno.EPIPE, 10053 is for Windows
logging.info ("Detected remote disconnect: " + host)
return
if code == errno.ECONNREFUSED:
logging.info ("Detected ECONNREFUSED: " + host)
return
if code in [54, 10054]: #reset
logging.info(host + ": reset from " + connectHost)
if not doProxy:
gConfig["BLOCKED_IPS"][connectHost] = True
return self.proxy()
print "error in proxy: ", self.requestline
print exc_type
print str(exc_value) + " " + host
# Send it down the pipe!
self._proxy_sock.sendall(self.mitm_request(req))
# Parse response
h = HTTPResponse(self._proxy_sock)
h.begin()
# Get rid of the pesky header
del h.msg['Transfer-Encoding']
# Time to relay the message across
res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason)
res += '%s\r\n' % h.msg
try:
res += h.read(h.length)
except IncompleteRead, e:
print e.message
# Let's close off the remote end
h.close()
self._proxy_sock.close()
# Relay the message
self.request.sendall(self.mitm_response(res))
def mitm_request(self, data):
for p in self.server._req_plugins:
data = p(self.server, self).do_request(data)
return data
def proxy(self):
doInject = False
try:
print self.requestline
self.supportCrLfPrefix = True
port = 80
host = self.headers["Host"]
if host.find(":") != -1:
port = int(host.split(":")[1])
host = host.split(":")[0]
# Remove http://[host]
path = self.path[self.path.find(host) + len(host) :]
connectHost = self.getip(host)
doInject = self.enableInjection(host, connectHost)
if self.remote is None or self.lastHost != self.headers["Host"]:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.connect((connectHost, port))
if doInject:
self.remote.send("\r\n\r\n")
self.lastHost = self.headers["Host"]
while True:
# Send requestline
self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n")
# Send headers
self.remote.send(str(self.headers) + "\r\n")
# Send Post data
if self.command == "POST":
self.remote.send(self.rfile.read(int(self.headers["Content-Length"])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
if response.status == 400 and self.supportCrLfPrefix == True:
while response.read(8192):
pass
self.supportCrLfPrefix = False
continue
break
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status + "\r\n")
h = ""
for hh, vv in response.getheaders():
if hh.upper() != "TRANSFER-ENCODING":
h += hh + ": " + vv + "\r\n"
self.wfile.write(h + "\r\n")
while True:
response_data = response.read(8192)
if len(response_data) == 0:
break
self.wfile.write(response_data)
except:
if self.remote:
self.remote.close()
self.remote = None
exc_type, exc_value, exc_traceback = sys.exc_info()
print "error in proxy: ", self.requestline
print exc_type
print exc_value
traceback.print_tb(exc_traceback)
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
if scm.upper() != "HTTP":
self.wfile.write("HTTP/1.1 500 Server Error " + scm.upper() + "\r\n")
elif netloc == urlparse.urlparse(PROXY_SERVER)[1]:
self.wfile.write("HTTP/1.1 500 Server Error, Cannot connect to proxy " + "\r\n")
else:
if doInject:
status = "HTTP/1.1 302 Found"
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + PROXY_SERVER + self.path[7:] + "\r\n")
else:
print ("Not redirect " + self.path)
self.wfile.write("HTTP/1.1 500 Server Error Unkown Error\r\n")
self.connection.close()
def proxy(self):
doProxy = False
inWhileList = False
logging.info (self.requestline)
port = 80
host = self.headers["Host"]
if host.find(":") != -1:
port = int(host.split(":")[1])
host = host.split(":")[0]
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
if host in ["127.0.0.1", "localhost"]:
basedir = os.path.dirname(__file__)
htmlTemplate = os.path.join(basedir, "index.html")
htmlFile = open(htmlTemplate)
html = htmlFile.read()
htmlFile.close()
status = "HTTP/1.1 200 OK"
if path == "/save":
postData = self.rfile.read(int(self.headers['Content-Length']))
data = urlparse.parse_qs(postData)
logging.info(str(data))
key = data["id"][0]
value = data["value"][0]
if key in gConfig:
if type(gConfig[key]) == type(True):
if value == "true": gConfig[key] = True
if value == "false": gConfig[key] = False
else:
gConfig[key] = type(gConfig[key]) (value)
self.wfile.write(status + "\r\n\r\n" + value)
return
if path == "/add":
postData = self.rfile.read(int(self.headers['Content-Length']))
data = urlparse.parse_qs(postData)
if "BLOCKED_DOMAINS" in data:
domain = data["BLOCKED_DOMAINS"][0]
if domain[:4] == "http":
(scm, netloc, path, params, query, _) = urlparse.urlparse(domain)
domain = netloc
gConfig["BLOCKED_DOMAINS"][domain] = True
self.wfile.write("HTTP/1.1 302 FOUND\r\n" + "Location: /\r\n\r\n" + domain)
return
#TODO: pac
for key in gConfig:
if type(gConfig[key]) in [str,int] :
html = html.replace("{"+key+"}", str(gConfig[key]))
else :
html = html.replace("{" + key + "}", str(gConfig[key]))
self.wfile.write(status + "\r\n\r\n" + html)
return
try:
if (gConfig["ADSHOSTON"] and host in gConfig["ADSHOST"]):
status = "HTTP/1.1 404 Not Found"
self.wfile.write(status + "\r\n\r\n")
return
# Remove http://[host] , for google.com.hk
path = self.path[self.path.find(netloc) + len(netloc):]
connectHost = self.getip(host)
logging.info ("Resolved " + host + " => " + connectHost)
if isDomainBlocked(host) or isIpBlocked(connectHost):
self.remote = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM)
logging.info("connect to " + host + ":" + str(port) + " var socks5 proxy")
self.remote.connect((connectHost, port))
doProxy = True
else:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logging.debug( "connect to " + host + ":" + str(port))
self.remote.connect((connectHost, port))
# Send requestline
if path == "":
path = "/"
print (" ".join((self.command, path, self.request_version)) + "\r\n")
self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n")
self.remote.send(str(self.headers) + "\r\n")
# Send Post data
if(self.command=='POST'):
self.remote.send(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
print (host + " response: %d"%(response.status))
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status + "\r\n")
h = ''
for hh, vv in response.getheaders():
if hh.upper()!='TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h + "\r\n")
dataLength = 0
while True:
response_data = response.read(8192)
if(len(response_data) == 0): break
if dataLength == 0 and (len(response_data) <= 501):
if response_data.find("<title>400 Bad Request") != -1 or response_data.find("<title>501 Method Not Implemented") != -1:
logging.error( host + " not supporting injection")
domainWhiteList.append(host)
response_data = gConfig["PAGE_RELOAD_HTML"]
self.wfile.write(response_data)
dataLength += len(response_data)
logging.debug( "data length: %d"%dataLength)
except:
if self.remote:
self.remote.close()
self.remote = None
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
status = "HTTP/1.1 302 Found"
exc_type, exc_value, exc_traceback = sys.exc_info()
if exc_type == socket.error:
code = exc_value[0]
if code == errno.EPIPE: #errno.EPIPE, 10053 is for Windows
logging.info ("Detected remote disconnect: " + host)
return
if code == errno.ECONNREFUSED:
logging.info ("Detected ECONNREFUSED: " + host)
return
if code in errno.ECONNRESET: #reset
logging.info(host + ": reset from " + connectHost)
if not doProxy:
gConfig["BLOCKED_IPS"][connectHost] = True
return self.proxy()
print ("error in proxy: ", self.requestline)
print (exc_type)
print (str(exc_value) + " " + host)
def mitm_response(self, data):
lines = data.split("\r\n")
r = HTTPResponse(FakeSocket(data))
r.begin()
# response line
self.doc.response.status = r.status
self.doc.response.responseline = lines[0]
# headers
ct = ""
cookies = list()
for header in r.getheaders():
name = header[0]
value = header[1]
self.doc.add_parsed_response_header(name, value)
if name == "content-type":
ct = value
elif name == "set-cookie":
cookies.append(value)
# content type
try:
m = reContentType.search(ct)
self.doc.response.content_type = m.group(1)
except:
pass
# cookies
for cookie in cookies:
# TODO: the following code extracts only partial cookie data - check/rewrite
try:
pc = SimpleCookie(cookie)
for name in pc.keys():
c = pc[name]
try:
value = c.value
except AttributeError:
value = None
try:
domain = c.domain
except AttributeError:
domain = None
try:
path = c.path
except AttributeError:
path = None
try:
exp = c.expires
except AttributeError:
exp = None
self.doc.add_response_cookie(name, value, domain, path, exp)
except:
pass
# body
bodybytes = r.read()
self.doc.response.body = bodybytes.decode(args.charset, args.encodingerrors)
try:
self.doc.save(storeResponseBody)
except:
pass
return data
def do_GET(self):
if self.path == 'http://weevely/':
self.send_cacert()
return
req = self
content_length = int(req.headers.get('Content-Length', 0))
req_body = self.rfile.read(content_length) if content_length else ''
if req.path[0] == '/':
if isinstance(self.connection, ssl.SSLSocket):
req.path = "https://%s%s" % (req.headers['Host'], req.path)
else:
req.path = "http://%s%s" % (req.headers['Host'], req.path)
req.headers['Content-length'] = str(len(req_body))
u = urlparse.urlsplit(req.path)
scheme, netloc, path = u.scheme, u.netloc, (u.path + '?' + u.query if u.query else u.path)
assert scheme in ('http', 'https')
if netloc:
req.headers['Host'] = netloc
setattr(req, 'headers', self.filter_headers(req.headers))
net_curl_args = [
'-X',
self.command,
'-i'
]
net_curl_args.append(self.path)
for h in req.headers:
if h.title().lower() == 'host':
host = self.headers[h]
net_curl_args += [ '-H', '%s: %s' % ( h.title(), self.headers[h] ) ]
if self.command == 'POST':
content_len = int(self.headers.getheader('content-length', 0))
net_curl_args += [ '-d', req_body ]
lock.acquire()
try:
result, headers, saved = ModuleExec(
'net_curl',
net_curl_args
).run()
finally:
lock.release()
if not headers:
log.debug('Error no headers')
self.send_error(502)
return
log.debug('> ' + '\r\n> '.join([ '%s: %s' % (h.title(), self.headers[h]) for h in self.headers ]))
log.debug('< ' + '\r\n< '.join(headers))
http_response_str = '\r\n'.join(headers) + '\r\n\r\n' + result
source = FakeSocket(http_response_str)
res = HTTPResponse(source)
res.begin()
version_table = {10: 'HTTP/1.0', 11: 'HTTP/1.1'}
setattr(res, 'headers', res.msg)
setattr(res, 'response_version', version_table[res.version])
# support streaming
if not 'Content-Length' in res.headers and 'no-store' in res.headers.get('Cache-Control', ''):
setattr(res, 'headers', self.filter_headers(res.headers))
self.relay_streaming(res)
return
try:
res_body = res.read()
except Exception as e:
log.debug(e)
self.send_error(500)
return
setattr(res, 'headers', self.filter_headers(res.headers))
self.wfile.write("%s %d %s\r\n" % (self.protocol_version, res.status, res.reason))
for line in res.headers.headers:
self.wfile.write(line)
self.end_headers()
self.wfile.write(res_body)
self.wfile.flush()
def _command(self, uri, method='GET', body='', **kwargs):
"""Makes an HTTP request through to an AirPlay server
Args:
uri(string): The URI to request
method(string): The HTTP verb to use when requesting `uri`, defaults to GET
body(string): If provided, will be sent witout alteration as the request body.
Content-Length header will be set to len(`body`)
**kwargs: If provided, Will be converted to a query string and appended to `uri`
Returns:
True: Request returned 200 OK, with no response body
False: Request returned something other than 200 OK, with no response body
Mixed: The body of the HTTP response
"""
# generate the request
if len(kwargs):
uri = uri + '?' + urlencode(kwargs)
request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str(len(body)) + "\r\n"
request+="Host: %s:%s\r\n" % (self.host,self.port)
request+="User-Agent: MediaControl/1.0\r\n"
request+="X-Apple-Session-ID: c6c0033e-96f9-11e6-b0a4-a45e60c9debb\r\n"
request+="Connection: close\r\n"
request+="\r\n"
request+=body
try:
if self.airplaySocket is None:
self.airplaySocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.airplaySocket.settimeout(self.timeout)
self.airplaySocket.connect((self.host, self.port))
except socket.error as exc:
self.airplaySocket=None
raise ValueError("Unable to connect to {0}:{1}: {2}".format(self.host, self.port, exc))
# send it
rs=self.airplaySocket.sendall(request)
# parse our response
result = self.airplaySocket.recv(self.RECV_SIZE)
resp = HTTPResponse(FakeSocket(result))
resp.begin()
# if our content length is zero, then return bool based on result code
if int(resp.getheader('content-length', 0)) == 0:
if resp.status == 200:
return True
else:
return False
# else, parse based on provided content-type
# and return the response body
content_type = resp.getheader('content-type')
if content_type is None:
raise RuntimeError('Response returned without a content type!')
return resp.read()
if 'WARC-Target-URI' in record:
lengths.append(len(record['WARC-Target-URI']))
content = record.payload.read()
src = FakeSocket(content)
response = HTTPResponse(src)
response.begin()
content_type = response.getheader('Content-Type')
if content_type is not None and "text/html" not in content_type:
print "Skipped content", response.getheader('Content-Type')
continue
#content_lengths.append(len(content))
try:
content = response.read()
content_lengths.append(len(content))
h = BeautifulSoup(content, "lxml")
except IncompleteRead:
continue
for title in h.findAll('title'):
print "TITLE", title.text.encode('utf-8')
for body in h.findAll("body"):
#text = [i for i in body.recursiveChildGenerator() if type(i) == NavigableString]
#text = [t.split() for t in text if len(t) > 0]
#print(text)
[s.extract() for s in body('script')]
[s.extract() for s in body('style')]
# Append message body if present to the request
if 'Content-Length' in self.headers:
req += self.rfile.read(int(self.headers['Content-Length']))
# Send it down the pipe!
self._proxy_sock.sendall(self.mitm_request(req))
# Parse response
h = HTTPResponse(self._proxy_sock)
h.begin()
# Get rid of the pesky header
del h.msg['Transfer-Encoding']
content = active_content_log(h.read(), h.msg, self.headers['Host'])
# Time to relay the message across
res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason)
res += '%s\r\n' % h.msg
res += content
# Let's close off the remote end
h.close()
self._proxy_sock.close()
# Relay the message
self.request.sendall(self.mitm_response(res))
def mitm_request(self, data):
for p in self.server._req_plugins:
def proxy(self):
doInject = False
try:
print self.requestline
port = 80
host = self.headers["Host"]
if host.find(":") != -1:
port = int(host.split(":")[1])
host = host.split(":")[0]
redirectUrl = self.path
while True:
(scm, netloc, path, params, query, _) = urlparse.urlparse(redirectUrl)
if (netloc not in gConfig["REDIRECT_DOMAINS"]):
break
prefixes = gConfig["REDIRECT_DOMAINS"][netloc].split('|')
found = False
for prefix in prefixes:
prefix = prefix + "="
for param in query.split('&') :
if param.find(prefix) == 0:
print "redirect to " + urllib.unquote(param[len(prefix):])
redirectUrl = urllib.unquote(param[len(prefix):])
found = True
continue
if not found:
break
#redirect
if (redirectUrl != self.path):
status = "HTTP/1.1 302 Found"
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + redirectUrl + "\r\n")
self.connection.close()
return
# Remove http://[host]
path = self.path[self.path.find(netloc) + len(netloc):]
connectHost = self.getip(host)
self.lastHost = self.headers["Host"]
while True:
doInject = self.enableInjection(host, connectHost)
if self.remote is None or self.lastHost != self.headers["Host"]:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print "connect to " + host + ":" + str(port)
self.remote.connect((connectHost, port))
if doInject:
self.remote.send("\r\n\r\n")
# Send requestline
self.remote.send(" ".join((self.command, (doInject and [path] or [self.path])[0], self.request_version)) + "\r\n")
# Send headers
self.remote.send(str(self.headers) + "\r\n")
# Send Post data
if(self.command=='POST'):
self.remote.send(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
badStatusLine = False
msg = "http405"
try :
response.begin()
print host + " response: %d"%(response.status)
msg = "http%d"%(response.status)
except BadStatusLine:
print host + " response: BadStatusLine"
msg = "badStatusLine"
badStatusLine = True
if doInject and (response.status == 400 or response.status == 405 or badStatusLine):
self.remote.close()
self.remote = None
domainWhiteList.append(host)
self.netlog(msg + "/host/" + host)
continue
break
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status + "\r\n")
h = ''
for hh, vv in response.getheaders():
if hh.upper()!='TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h + "\r\n")
while True:
response_data = response.read(8192)
if(len(response_data) == 0): break
self.wfile.write(response_data)
except:
if self.remote:
self.remote.close()
self.remote = None
exc_type, exc_value, exc_traceback = sys.exc_info()
print "error in proxy: ", self.requestline
print exc_type
print str(exc_value) + " " + host
path = ""
if exc_type == socket.error:
code, msg = str(exc_value).split('] ')
code = code[1:].replace(" ", "")
path = code + "/host/" + host + "/?msg=" + urllib.quote(msg)
traceback.print_tb(exc_traceback)
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
status = "HTTP/1.1 302 Found"
if (netloc != urlparse.urlparse( gConfig["PROXY_SERVER"] )[1]):
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + gConfig["PROXY_SERVER"] + self.path[7:] + "\r\n")
else:
status = "HTTP/1.1 302 Found"
if (scm.upper() != "HTTP"):
msg = "schme-not-supported"
else:
msg = "web-proxy-fail"
path = ("error/host/" + host + "/?msg=" + msg)
self.wfile.write(status + "\r\n")
self.wfile.write("Location: http://liruqi.info/post/18486575704/west-chamber-proxy#" + msg + "\r\n")
self.netlog(path)
self.connection.close()
def proxy(self):
inWhileList = False
logging.info (self.requestline)
port = 80
host = self.headers["Host"]
if host.find(":") != -1:
port = int(host.split(":")[1])
host = host.split(":")[0]
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
if host in ["127.0.0.1", "localhost"]:
basedir = os.path.dirname(__file__)
htmlTemplate = os.path.join(basedir, "index.html")
htmlFile = open(htmlTemplate)
html = htmlFile.read()
htmlFile.close()
status = "HTTP/1.1 200 OK"
if path == "/save":
postData = self.rfile.read(int(self.headers['Content-Length']))
data = urlparse.parse_qs(postData)
logging.info(str(data))
key = data["id"][0]
value = data["value"][0]
if key in gConfig:
if type(gConfig[key]) == type(True):
if value == "true": gConfig[key] = True
if value == "false": gConfig[key] = False
else:
gConfig[key] = type(gConfig[key]) (value)
self.wfile.write(status + "\r\n\r\n" + value)
return
if path == "/add":
postData = self.rfile.read(int(self.headers['Content-Length']))
data = urlparse.parse_qs(postData)
if "BLOCKED_DOMAINS" in data:
domain = data["BLOCKED_DOMAINS"][0]
if domain[:4] == "http":
(scm, netloc, path, params, query, _) = urlparse.urlparse(domain)
domain = netloc
gConfig["BLOCKED_DOMAINS"][domain] = True
self.wfile.write("HTTP/1.1 302 FOUND\r\n" + "Location: /\r\n\r\n" + domain)
return
for key in gConfig:
if type(gConfig[key]) in [str,int] :
html = html.replace("{"+key+"}", str(gConfig[key]))
else :
html = html.replace("{" + key + "}", str(gConfig[key]))
self.wfile.write(status + "\r\n\r\n" + html)
return
try:
if (host in gConfig["HSTS_DOMAINS"]):
redirectUrl = "https://" + self.path[7:]
#redirect
status = "HTTP/1.1 302 Found"
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + redirectUrl + "\r\n\r\n")
return
for d in domainWhiteList:
if host.endswith(d):
logging.info (host + " in domainWhiteList: " + d)
inWhileList = True
connectHost = self.getip(host)
# Remove http://[host] , for google.com.hk
path = self.path[self.path.find(netloc) + len(netloc):]
if path == "":
path = "/"
if isDomainBlocked(host) or isIpBlocked(connectHost):
if gConfig['PROXY_TYPE'] == 'socks5':
self.remote = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM)
else:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connectHost = gConfig['HTTP_PROXY']
port = gConfig['HTTP_PROXY_PORT']
path = self.path
else:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logging.debug( host + ":connect to " + connectHost + ":" + str(port))
self.remote.connect((connectHost, port))
print " ".join((self.command, self.path, self.request_version)) + "\r\n"
self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n" + str(self.headers) + "\r\n")
# Send Post data
if(self.command=='POST'):
self.remote.send(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
badStatusLine = False
try :
response.begin()
print host + " response: %d"%(response.status)
except:
self.remote.close()
self.remote = None
raise
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status + "\r\n")
h = ''
for hh, vv in response.getheaders():
if hh.upper()!='TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h + "\r\n")
dataLength = 0
while True:
response_data = response.read(8192)
if(len(response_data) == 0): break
self.wfile.write(response_data)
dataLength += len(response_data)
logging.debug( "data length: %d"%dataLength)
self.remote.close()
self.remote = None
except:
if self.remote:
self.remote.close()
self.remote = None
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
status = "HTTP/1.1 302 Found"
if host in gConfig["HSTS_ON_EXCEPTION_DOMAINS"]:
redirectUrl = "https://" + self.path[7:]
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + redirectUrl + "\r\n")
exc_type, exc_value, exc_traceback = sys.exc_info()
if exc_type == socket.error:
code = exc_value[0]
if code in [32, 10053]: #errno.EPIPE, 10053 is for Windows
logging.info ("Detected remote disconnect: " + host)
return
if code in [54, 10054]: #reset
logging.info(host + ": reset from " + connectHost)
gConfig["BLOCKED_IPS"][connectHost] = True
return
print "error in proxy: ", self.requestline, exc_type
print str(exc_value) + " " + host
if exc_type == socket.timeout or (exc_type == socket.error and code in [60, 110, 10060]): #timed out, 10060 is for Windows
if not inWhileList:
logging.info ("add "+host+" to blocked domains")
gConfig["BLOCKED_IPS"][connectHost] = True
def proxy(self):
doInject = False
inWhileList = False
logging.info(self.requestline)
port = 80
host = self.headers["Host"]
if host.find(":") != -1:
port = int(host.split(":")[1])
host = host.split(":")[0]
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
if host in ["127.0.0.1", "localhost"]:
basedir = os.path.dirname(__file__)
htmlTemplate = os.path.join(basedir, "index.html")
htmlFile = open(htmlTemplate)
html = htmlFile.read()
htmlFile.close()
status = "HTTP/1.1 200 OK"
if path == "/save":
postData = self.rfile.read(int(self.headers['Content-Length']))
data = urlparse.parse_qs(postData)
logging.info(str(data))
key = data["id"][0]
value = data["value"][0]
if key in gConfig:
if type(gConfig[key]) == type(True):
if value == "true": gConfig[key] = True
if value == "false": gConfig[key] = False
else:
gConfig[key] = type(gConfig[key])(value)
hookInit()
self.wfile.write(status + "\r\n\r\n" + value)
return
if path == "/add":
postData = self.rfile.read(int(self.headers['Content-Length']))
data = urlparse.parse_qs(postData)
if "BLOCKED_DOMAINS" in data:
domain = data["BLOCKED_DOMAINS"][0]
if domain[:4] == "http":
(scm, netloc, path, params, query,
_) = urlparse.urlparse(domain)
domain = netloc
gConfig["BLOCKED_DOMAINS"][domain] = True
self.wfile.write("HTTP/1.1 302 FOUND\r\n" +
"Location: /\r\n\r\n" + domain)
return
for key in gConfig:
if type(gConfig[key]) in [str, int]:
html = html.replace("{" + key + "}", str(gConfig[key]))
else:
html = html.replace("{" + key + "}", str(gConfig[key]))
self.wfile.write(status + "\r\n\r\n" + html)
return
try:
if (host in gConfig["HSTS_DOMAINS"]):
redirectUrl = "https://" + self.path[7:]
#redirect
status = "HTTP/1.1 302 Found"
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + redirectUrl + "\r\n\r\n")
return
if (gConfig["ADSHOSTON"] and host in gConfig["ADSHOST"]):
status = "HTTP/1.1 404 Not Found"
self.wfile.write(status + "\r\n\r\n")
return
# Remove http://[host] , for google.com.hk
path = self.path[self.path.find(netloc) + len(netloc):]
for d in domainWhiteList:
if host.endswith(d):
logging.info(host + " in domainWhiteList: " + d)
inWhileList = True
connectHost = self.getip(host)
if not inWhileList:
doInject = self.enableInjection(host, connectHost)
logging.info("Resolved " + host + " => " + connectHost)
if isDomainBlocked(host) or isIpBlocked(connectHost):
if gConfig["PROXY_TYPE"] == "socks5":
self.remote = socks.socksocket(socket.AF_INET,
socket.SOCK_STREAM)
logging.info("connect to " + host + ":" + str(port) +
" var socks5 proxy")
self.remote.connect((connectHost, port))
else:
logging.info(host + " blocked, try goagent.")
return self.do_METHOD_Tunnel()
else:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logging.debug("connect to " + host + ":" + str(port))
self.remote.connect((connectHost, port))
if doInject:
logging.info("inject http for " + host)
self.remote.send("\r\n\r\n")
# Send requestline
if path == "":
path = "/"
print " ".join((self.command, path, self.request_version)) + "\r\n"
self.remote.send(" ".join((self.command, path,
self.request_version)) + "\r\n")
self.remote.send(str(self.headers) + "\r\n")
# Send Post data
if (self.command == 'POST'):
self.remote.send(
self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
badStatusLine = False
msg = "http405"
try:
response.begin()
print host + " response: %d" % (response.status)
msg = "http%d" % (response.status)
except BadStatusLine:
print host + " response: BadStatusLine"
msg = "badStatusLine"
badStatusLine = True
except:
raise
if doInject and (response.status == 400 or response.status == 405
or badStatusLine):
self.remote.close()
self.remote = None
logging.info(host + " seem not support inject, " + msg)
return self.do_METHOD_Tunnel()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status + "\r\n")
h = ''
for hh, vv in response.getheaders():
if hh.upper() != 'TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h + "\r\n")
dataLength = 0
while True:
response_data = response.read(8192)
if (len(response_data) == 0): break
if dataLength == 0 and (len(response_data) <= 501):
if response_data.find(
"<title>400 Bad Request"
) != -1 or response_data.find(
"<title>501 Method Not Implemented") != -1:
logging.error(host + " not supporting injection")
domainWhiteList.append(host)
response_data = gConfig["PAGE_RELOAD_HTML"]
self.wfile.write(response_data)
dataLength += len(response_data)
logging.debug("data length: %d" % dataLength)
except:
if self.remote:
self.remote.close()
self.remote = None
(scm, netloc, path, params, query,
_) = urlparse.urlparse(self.path)
status = "HTTP/1.1 302 Found"
if host in gConfig["HSTS_ON_EXCEPTION_DOMAINS"]:
redirectUrl = "https://" + self.path[7:]
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + redirectUrl + "\r\n")
exc_type, exc_value, exc_traceback = sys.exc_info()
if exc_type == socket.error:
code, msg = str(exc_value).split('] ')
code = code[1:].split(' ')[1]
if code in ["32", "10053"]: #errno.EPIPE, 10053 is for Windows
logging.info("Detected remote disconnect: " + host)
return
if code in ["54"]:
return self.do_METHOD_Tunnel()
if code in ["61"]: #server not support injection
if doInject:
logging.info("try not inject " + host)
domainWhiteList.append(host)
return self.do_METHOD_Tunnel()
print "error in proxy: ", self.requestline
print exc_type
print str(exc_value) + " " + host
if exc_type == socket.timeout or (
exc_type == socket.error
and code in ["60", "110", "10060"
]): #timed out, 10060 is for Windows
if not inWhileList:
logging.info("add " + host + " to blocked domains")
gConfig["BLOCKED_DOMAINS"][host] = True
return self.do_METHOD_Tunnel()
def proxy(self):
doInject = False
if gOptions.log > 0: print self.requestline
port = 80
host = self.headers["Host"]
if host.find(":") != -1:
port = int(host.split(":")[1])
host = host.split(":")[0]
errpath = ""
try:
redirectUrl = self.path
while True:
(scm, netloc, path, params, query, _) = urlparse.urlparse(redirectUrl)
if gOptions.log > 2: print urlparse.urlparse(redirectUrl)
if (netloc not in gConfig["REDIRECT_DOMAINS"]):
break
prefixes = gConfig["REDIRECT_DOMAINS"][netloc].split('|')
found = False
for prefix in prefixes:
prefix = prefix + "="
for param in query.split('&') :
if param.find(prefix) == 0:
print "redirect to " + urllib.unquote(param[len(prefix):])
redirectUrl = urllib.unquote(param[len(prefix):])
found = True
continue
if not found:
break
if (host in gConfig["HSTS_DOMAINS"]):
redirectUrl = "https://" + self.path[7:]
#redirect
if (redirectUrl != self.path):
status = "HTTP/1.1 302 Found"
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + redirectUrl + "\r\n")
self.wfile.close()
return
# Remove http://[host] , for google.com.hk
path = self.path[self.path.find(netloc) + len(netloc):]
if host in gConfig["BLOCKED_DOMAINS"]:
host = gConfig["PROXY_SERVER_SIMPLE"]
path = self.path[len(scm)+2:]
self.headers["Host"] = gConfig["PROXY_SERVER_SIMPLE"]
print "use simple web proxy for " + path
self.lastHost = self.headers["Host"]
while True:
inWhileList = False
for d in domainWhiteList:
if host.endswith(d):
if gOptions.log > 1: print host + " in domainWhiteList: " + d
inWhileList = True
connectHost = host
if not inWhileList:
connectHost = self.getip(host)
if connectHost in gConfig["BLOCKED_IPS"]:
gConfig["BLOCKED_DOMAINS"][host] = True
host = gConfig["PROXY_SERVER_SIMPLE"]
path = self.path[len(scm)+2:]
self.headers["Host"] = gConfig["PROXY_SERVER_SIMPLE"]
connectHost = self.getip(host)
print "use simple web proxy for " + path
doInject = self.enableInjection(host, connectHost)
if self.remote is None or self.lastHost != self.headers["Host"]:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if gOptions.log > 1: print "connect to " + host + ":" + str(port)
self.remote.connect((connectHost, port))
if doInject:
if gOptions.log > 0: print "inject http for "+host
self.remote.send("\r\n\r\n")
# Send requestline
if path == "":
path = "/"
print " ".join((self.command, path, self.request_version)) + "\r\n"
self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n")
# Send headers
self.remote.send(str(self.headers) + "\r\n")
# Send Post data
if(self.command=='POST'):
self.remote.send(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
badStatusLine = False
msg = "http405"
try :
response.begin()
print host + " response: %d"%(response.status)
msg = "http%d"%(response.status)
except BadStatusLine:
print host + " response: BadStatusLine"
msg = "badStatusLine"
badStatusLine = True
except:
raise
if doInject and (response.status == 400 or response.status == 405 or badStatusLine) and host != gConfig["PROXY_SERVER_SIMPLE"]:
self.remote.close()
self.remote = None
domainWhiteList.append(host)
errpath = (msg + "/host/" + host)
continue
break
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status + "\r\n")
h = ''
for hh, vv in response.getheaders():
if hh.upper()!='TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h + "\r\n")
dataLength = 0
while True:
response_data = response.read(8192)
if(len(response_data) == 0): break
if dataLength == 0 and (len(response_data) <= 320):
if response_data.find("<title>400 Bad Request") != -1 or response_data.find("<title>501 Method Not Implemented") != -1:
print host + " not supporting injection"
domainWhiteList.append(host)
response_data = gConfig["PAGE_RELOAD_HTML"]
self.wfile.write(response_data)
dataLength += len(response_data)
if gOptions.log > 1: print "data length: %d"%dataLength
self.wfile.close()
except:
if self.remote:
self.remote.close()
self.remote = None
exc_type, exc_value, exc_traceback = sys.exc_info()
if exc_type == socket.error:
code, msg = str(exc_value).split('] ')
code = code[1:].split(' ')[1]
if code in ["32", "10053"]: #errno.EPIPE, 10053 is for Windows
if gOptions.log > 0: print "Detected remote disconnect: " + host
self.wfile.close()
return
if code in ["61"]: #server not support injection
if doInject:
print "try not inject " + host
domainWhiteList.append(host)
self.proxy()
return
print "error in proxy: ", self.requestline
print exc_type
print str(exc_value) + " " + host
errpath = "unkown/host/" + host
if exc_type == socket.timeout or (exc_type == socket.error and code in ["60", "110", "10060"]): #timed out, 10060 is for Windows
if gOptions.log > 0: print "add "+host+" to blocked domains"
gConfig["BLOCKED_DOMAINS"][host] = True
self.wfile.write("HTTP/1.1 200 OK\r\n\r\n")
self.wfile.write(gConfig["PAGE_RELOAD_HTML"])
return
traceback.print_tb(exc_traceback)
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
status = "HTTP/1.1 302 Found"
if (netloc != urlparse.urlparse( gConfig["PROXY_SERVER"] )[1] and doInject):
self.wfile.write(status + "\r\n")
redirectUrl = gConfig["PROXY_SERVER"] + self.path[7:]
if host in gConfig["HSTS_ON_EXCEPTION_DOMAINS"]:
redirectUrl = "https://" + self.path[7:]
self.wfile.write("Location: " + redirectUrl + "\r\n")
else:
if (scm.upper() != "HTTP"):
msg = "schme-not-supported"
else:
msg = "web-proxy-fail"
errpath = ("error/host/" + host + "/?msg=" + msg)
self.wfile.write(status + "\r\n")
self.wfile.write("Location: http://westchamberproxy.appspot.com/#" + msg + "\r\n")
self.wfile.close()
print "client connection closed"
if errpath != "":
self.netlog(errpath)
def do_COMMAND(self):
global allCache
global dumpToDisk
cacheHit = False
sendNotModifiesResponse = False
cacheurl = self.path
print cacheurl
# checking if we have a cache hit
if not ('?' in cacheurl) and (cacheurl in allCache):
#in all cache array add two more fields max-age and date,time when added, and check if the object is valid before
#making cachehit true
cacheHit = True
#------ code below uncomment
# if 'If-Modified-Since' in self.headers:
# last_modified_date = self.headers['If-Modified-Since']
# formatter_string = "%a, %d %b %Y %H:%M:%S %Z"
# in_req_modified_date = datetime.strptime(last_modified_date, formatter_string)
# #get the other date from cache
# in_cache_modified_date = allCache[cacheurl][3]
# if in_req_modified_date > in_cache_modified_date:
# #can be served from cache
sendNotModifiesResponse = False
# No cache hit, fetch from Internet
if not cacheHit:
# Is this an SSL tunnel?
if not self.is_connect:
try:
# Connect to destination
self._connect_to_host()
except Exception, e:
self.send_error(500, str(e))
return
# Extract path
# Build request
req = '%s %s %s\r\n' % (self.command, self.path, self.request_version)
# Add headers to the request
req += '%s\r\n' % self.headers
# Append message body if present to the request
if 'Content-Length' in self.headers:
req += self.rfile.read(int(self.headers['Content-Length']))
# Send it down the pipe!
self._proxy_sock.sendall(self.mitm_request(req))
# Parse response
h = HTTPResponse(self._proxy_sock)
h.begin()
# Get rid of the pesky header
del h.msg['Transfer-Encoding']
# Time to relay the message across
res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason)
res += '%s\r\n' % h.msg
toWrite=h.read()
res += toWrite
# fetched an object from the Internet, need to add it to memory and dump it to disk
if not ("?" in cacheurl):
fileName=cacheFileName()
tempObject=CacheObject(cacheurl, h.getheaders(), h.status, h.reason, toWrite,fileName)
#
current_datetime = datetime.now()
#
allCache[cacheurl]=['memory', tempObject, fileName, current_datetime]
dumpToDisk.append(tempObject)
h.close()
self._proxy_sock.close()
if 'Content-Length' in self.headers:
req += self.rfile.read(int(self.headers['Content-Length']))
try:
# Send it down the pipe!
self._proxy_sock.sendall(self.mitm_request(req))
# Parse response
h = HTTPResponse(self._proxy_sock)
h.begin()
# Get rid of the pesky header
del h.msg['Transfer-Encoding']
# Time to relay the message across
res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason)
res += '%s\r\n' % h.msg
content_received = h.read()
res += content_received
#log the size
try:
http_obj = recorder.http_object(h.getheaders(), url_requested,
content_received, h.status,
h.reason, self.request_version,
webpage, self.rtt)
recorder.save_obj(http_obj)
except Exception, e:
print str(e)
self.request.sendall(self.mitm_response(res))
except SocketError as e:
if e.errno != errno.ECONNRESET:
def mitm_response(self, data):
lines = data.split("\r\n")
r = HTTPResponse(FakeSocket(data))
r.begin()
# response line
self.doc.response.status = r.status
self.doc.response.responseline = lines[0].decode(args.charset, args.encodingerrors)
# headers
ct = ""
cookies = list()
for header in r.getheaders():
name = header[0].decode(args.charset, args.encodingerrors)
value = header[1].decode(args.charset, args.encodingerrors)
self.doc.add_parsed_response_header(name, value)
if name == "content-type":
ct = value
elif name == "set-cookie":
cookies.append(value)
# content type
try:
m = reContentType.search(ct)
self.doc.response.content_type = m.group(1)
except:
pass
# cookies
for cookie in cookies:
# TODO: the following code extracts only partial cookie data - check/rewrite
try:
pc = SimpleCookie(cookie)
for name in pc.keys():
c = pc[name]
try:
value = c.value
except AttributeError:
value = None
try:
domain = c.domain
except AttributeError:
domain = None
try:
path = c.path
except AttributeError:
path = None
try:
exp = c.expires
except AttributeError:
exp = None
self.doc.add_response_cookie(name, value, domain, path, exp)
except:
pass
# body
bodybytes = r.read()
self.doc.response.body = bodybytes.decode(args.charset, args.encodingerrors)
self.doc.save(storeResponseBody)
return data
def proxy(self):
doInject = False
inWhileList = False
logging.info (self.requestline)
port = 80
host = self.headers["Host"]
if host.find(":") != -1:
port = int(host.split(":")[1])
host = host.split(":")[0]
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
if host in ["127.0.0.1", "localhost"]:
basedir = os.path.dirname(__file__)
htmlTemplate = os.path.join(basedir, "index.html")
htmlFile = open(htmlTemplate)
html = htmlFile.read()
htmlFile.close()
status = "HTTP/1.1 200 OK"
if path == "/save":
postData = self.rfile.read(int(self.headers['Content-Length']))
data = urlparse.parse_qs(postData)
logging.info(str(data))
key = data["id"][0]
value = data["value"][0]
if key in gConfig:
if type(gConfig[key]) == type(True):
if value == "true": gConfig[key] = True
if value == "false": gConfig[key] = False
else:
gConfig[key] = type(gConfig[key]) (value)
hookInit()
self.wfile.write(status + "\r\n\r\n" + value)
return
for key in gConfig:
if type(gConfig[key]) in [str,int] :
html = html.replace("{"+key+"}", str(gConfig[key]))
else :
html = html.replace("{" + key + "}", str(gConfig[key]))
self.wfile.write(status + "\r\n\r\n" + html)
return
try:
if (host in gConfig["HSTS_DOMAINS"]):
redirectUrl = "https://" + self.path[7:]
#redirect
status = "HTTP/1.1 302 Found"
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + redirectUrl + "\r\n\r\n")
return
if (gConfig["ADSHOSTON"] and host in gConfig["ADSHOST"]):
status = "HTTP/1.1 404 Not Found"
self.wfile.write(status + "\r\n\r\n")
return
# Remove http://[host] , for google.com.hk
path = self.path[self.path.find(netloc) + len(netloc):]
connectHost = self.getip(host)
logging.info ("Resolved " + host + " => " + connectHost)
rootDomain = string.join(host.split('.')[-2:], '.')
if True:
for d in domainWhiteList:
if host.endswith(d):
logging.info (host + " in domainWhiteList: " + d)
inWhileList = True
if not inWhileList:
doInject = self.enableInjection(host, connectHost)
if isDomainBlocked(host) or isIpBlocked(connectHost):
if gConfig["PROXY_TYPE"] == "socks5":
self.remote = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM)
logging.info("connect to " + host + ":" + str(port) + " var socks5 proxy")
self.remote.connect((connectHost, port))
else:
logging.info(host + " blocked, try goagent.")
return self.do_METHOD_Tunnel()
else:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logging.debug( "connect to " + host + ":" + str(port))
self.remote.connect((connectHost, port))
if doInject:
logging.info ("inject http for "+host)
self.remote.send("\r\n\r\n")
# Send requestline
if path == "":
path = "/"
print " ".join((self.command, path, self.request_version)) + "\r\n"
self.remote.send(" ".join((self.command, path, self.request_version)) + "\r\n")
self.remote.send(str(self.headers) + "\r\n")
# Send Post data
if(self.command=='POST'):
self.remote.send(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
badStatusLine = False
msg = "http405"
try :
response.begin()
print host + " response: %d"%(response.status)
msg = "http%d"%(response.status)
except BadStatusLine:
print host + " response: BadStatusLine"
msg = "badStatusLine"
badStatusLine = True
except:
raise
if doInject and (response.status == 400 or response.status == 405 or badStatusLine):
self.remote.close()
self.remote = None
logging.info (host + " seem not support inject, " + msg)
return self.do_METHOD_Tunnel()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status + "\r\n")
h = ''
for hh, vv in response.getheaders():
if hh.upper()!='TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h + "\r\n")
dataLength = 0
while True:
response_data = response.read(8192)
if(len(response_data) == 0): break
if dataLength == 0 and (len(response_data) <= 501):
if response_data.find("<title>400 Bad Request") != -1 or response_data.find("<title>501 Method Not Implemented") != -1:
logging.error( host + " not supporting injection")
domainWhiteList.append(host)
response_data = gConfig["PAGE_RELOAD_HTML"]
self.wfile.write(response_data)
dataLength += len(response_data)
logging.debug( "data length: %d"%dataLength)
except:
if self.remote:
self.remote.close()
self.remote = None
(scm, netloc, path, params, query, _) = urlparse.urlparse(self.path)
status = "HTTP/1.1 302 Found"
if host in gConfig["HSTS_ON_EXCEPTION_DOMAINS"]:
redirectUrl = "https://" + self.path[7:]
self.wfile.write(status + "\r\n")
self.wfile.write("Location: " + redirectUrl + "\r\n")
exc_type, exc_value, exc_traceback = sys.exc_info()
if exc_type == socket.error:
code, msg = str(exc_value).split('] ')
code = code[1:].split(' ')[1]
if code in ["32", "10053"]: #errno.EPIPE, 10053 is for Windows
logging.info ("Detected remote disconnect: " + host)
return
if code in ["61"]: #server not support injection
if doInject:
logging.info( "try not inject " + host)
domainWhiteList.append(host)
self.do_METHOD_Tunnel()
return
print "error in proxy: ", self.requestline
print exc_type
print str(exc_value) + " " + host
if exc_type == socket.timeout or (exc_type == socket.error and code in ["60", "110", "10060"]): #timed out, 10060 is for Windows
if not inWhileList:
logging.info ("add "+host+" to blocked domains")
gConfig["BLOCKED_DOMAINS"][host] = True
return self.do_METHOD_Tunnel()
try:
# Send it down the pipe!
self._proxy_sock.sendall(self.mitm_request(req))
# Parse response
h = HTTPResponse(self._proxy_sock)
h.begin()
# Get rid of the pesky header
del h.msg['Transfer-Encoding']
# Time to relay the message across
res = '%s %s %s\r\n' % (self.request_version, h.status, h.reason)
res += '%s\r\n' % h.msg
content_received = h.read()
res += content_received
HTTPObject_received = controller.HTTPObject(h.getheaders(), url_requested , content_received, h.status)
controller.createObject(HTTPObject_received)
# Relay the message
self.request.sendall(self.mitm_response(res))
except SocketError as e:
if e.errno != errno.ECONNRESET:
raise
pass
# Let's close off the remote end
if h != None:
