def _on_request(self,
urlopen,
path,
pool,
method,
url,
body=None,
headers=None,
**kw):
# Remove bypass headers
real_headers = dict(headers or {})
real_headers.pop(URLLIB3_BYPASS)
# Create request contract based on incoming params
req = Request(method)
req.headers = real_headers
req.body = body
# Compose URL
req.url = '{}://{}:{:d}{}'.format(pool.scheme, pool.host, pool.port
or 80, url)
# Match the request against the registered mocks in pook
mock = self.engine.match(req)
# If cannot match any mock, run real HTTP request since networking
# or silent model will be enabled, otherwise this statement won't
# be reached (an exception will be raised before).
if not mock:
return urlopen(pool, method, url, body=body, headers=headers, **kw)
# Shortcut to mock response and response body
res = mock._response
body = res._body
# Aggregate headers as list of tuples for interface compatibility
headers = []
for key in res._headers:
headers.append((key, res._headers[key]))
if is_chunked_response(headers):
body_chunks = body if isinstance(body, list) else [body]
body_chunks = [chunk.encode() for chunk in body_chunks]
body = ClientHTTPResponse(MockSock)
body.fp = FakeChunkedResponseBody(body_chunks)
else:
# Assume that the body is a bytes-like object
body = body_io(body)
# Return mocked HTTP response
return HTTPResponse(
path,
body=body,
status=res._status,
headers=headers,
preload_content=False,
reason=http_reasons.get(res._status),
original_response=FakeResponse(method, headers),
)
def testPipelining(self):
# Tests the use of several requests issued at once.
s = ("GET / HTTP/1.0\r\n"
"Connection: %s\r\n"
"Content-Length: %d\r\n"
"\r\n"
"%s")
to_send = ''
count = 25
for n in range(count):
body = "Response #%d\r\n" % (n + 1)
if n + 1 < count:
conn = 'keep-alive'
else:
conn = 'close'
to_send += s % (conn, len(body), body)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(to_send)
for n in range(count):
expect_body = "Response #%d\r\n" % (n + 1)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(response_body))
self.failUnlessEqual(response_body, expect_body)
def proxy(self):
if self.remote is None or self.lastHost != self.headers['Host']:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.connect((proxy_host, proxy_port))
self.remote.sendall(self.requestline.encode('ascii') + b'\r\n')
# Add auth inf.
self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(auth)
#self.remote.sendall('Proxy-Authorization: Basic %s\r\n' % base64.b64encode(auth))
headerstr = str(self.headers).replace('\r\n', '\n').replace('\n', '\r\n')
self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
# Send Post data
if self.command == 'POST':
self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
# Reply to the browser
status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
hlist = []
for line in response.msg.headers: # Fixed multiple values of a same name
if 'TRANSFER-ENCODING' not in line.upper():
hlist.append(line)
self.wfile.write(''.join(hlist) + b'\r\n')
if self.command == 'CONNECT' and response.status == 200:
return self.transfer(self.remote, self.connection)
else:
while True:
response_data = response.read(BufferSize)
if not response_data: break
self.wfile.write(response_data)
def parseNewResponse(self, packetString, src, dst, tcp):
responseCode = packetString[9:12]
if responseCode == '200': # just okay responses for now
if '\r\n\r\n' in packetString: # only proceed if the response has a body
bodyIndex = packetString.index('\r\n\r\n') + 4
body = packetString[bodyIndex:]
socket = FakeSocket(packetString)
response = HTTPResponse(socket)
response.begin()
headerArray = response.getheaders()
for item in headerArray:
flowKey = (src, dst)
if item[0] == 'content-type' and 'text/html' in item[1]: # accept any kind of text content
print headerArray
for item in headerArray:
if item[0] == 'content-length':
print 'found fixed length'
length = int(item[1])
if length is not 0:
self.parseFixedLengthResponse(flowKey, body, length, src, dst, tcp, responseCode)
else:
print "warning, content-length is zero!"
elif item[0] == 'transfer-encoding' and item[1] == 'chunked':
print 'found chunked'
self.parseChunkedResponse(flowKey, body, src, dst, tcp, responseCode)
else:
print "body not found"
def sogouProxy(self):
if self.headers["Host"].startswith('chrome_dcp_proxy_pac.cnbeining'): #Give a PAC file
self.wfile.write("HTTP/1.1 200 OK".encode('ascii') + b'\r\n')
hstr = '''Host: 127.0.0.1
function FindProxyForURL(url, host) {
if (url.substring(0,5) == 'http:' &&
!isPlainHostName(host) &&
!shExpMatch(host, '*.local') &&
!isInNet(dnsResolve(host), '10.0.0.0', '255.0.0.0') &&
!isInNet(dnsResolve(host), '172.16.0.0', '255.240.0.0') &&
!isInNet(dnsResolve(host), '192.168.0.0', '255.255.0.0') &&
!isInNet(dnsResolve(host), '127.0.0.0', '255.255.255.0') )
return 'PROXY ''' + server_ip + ':' + str(server_port) + '''; DIRECT';
return 'DIRECT';
}'''
self.wfile.write(hstr + b'\r\n')
return
if self.remote is None or self.lastHost != self.headers["Host"]:
if PROXY_MODE == 'HTTPS':
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
context.verify_mode = ssl.CERT_REQUIRED
context.check_hostname = True
context.load_default_certs()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(RemoteTimeout)
self.remote = context.wrap_socket(s, server_hostname='proxy.googlezip.net')
self.remote.connect(('proxy.googlezip.net', 443))
else: #HTTP
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.settimeout(RemoteTimeout)
self.remote.connect(("compress.googlezip.net", 80))
self.remote.sendall(self.requestline.encode('ascii') + b"\r\n")
# Add Verification Tags
self.headers["Chrome-Proxy"] = get_google_header()
headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
# Send Post data
if self.command == 'POST':
self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
hlist = []
for line in response.msg.headers: # Fixed multiple values of a same name
if 'TRANSFER-ENCODING' not in line.upper():
hlist.append(line)
self.wfile.write("".join(hlist) + b'\r\n')
if self.command == "CONNECT": # NO HTTPS, as Chrome DCP does not allow HTTPS traffic
return
else:
while True:
response_data = response.read(BufferSize)
if not response_data: break
self.wfile.write(response_data)
def get_json(self, response):
res_str = str(helpers.bytesToString(response.getResponse()))
source = FakeSocket(res_str)
res_obj = HTTPResponse(source)
res_obj.begin()
print "Reading response object"
return json.loads(res_obj.read())
def send_request(self, request):
self.request_num += 1
# Sanity check: if we're sending a request with a content-length but
# we don't have that many bytes to send, we'll just get a 504. Don't
# send it and instead report a client error.
parts = request.split('\r\n\r\n', 1)
if len(parts) > 1:
req, body = parts
match = content_length_re.search(req)
if match:
if len(body) < int(match.group(1)):
self.error("request body of incorrect size")
return True
if not self.connection:
self._connect()
if self.connection:
# tally request method
#self.tally(request.split(" ", 1)[0])
try:
self.connection.sendall(request)
response = HTTPResponse(self.connection)
response.begin()
self.tally(response.status)
while response.read():
pass
if response.will_close:
# We hope our Connection: keep-alive won't be ignored, but
# deal with it if it does.
self._disconnect()
if self.options.speedup < 0.8:
# if we're slowing down by too much, keep-alive will just
# result in the server getting bored between requests and
# dropping the connection, so disable it.
self._disconnect()
return True
except IncompleteRead:
self.error("error while reading response: IncompleteRead (terminating job)")
self._disconnect()
except Exception, e: # TODO: more restrictive error catching?
self.error("error while sending request and reading response: %s %s" % (type(e), e))
self._disconnect()
if self.connection:
self.connection.close()
self.connection = None
def do_proxy(self):
try:
if self.s == 0:
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((proxy_host, proxy_port))
self.s.send(self.requestline.encode('ascii') + b'\r\n')
# Add Sogou Verification Tags
self.headers['X-Sogou-Auth'] = x_sogou_auth
t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
self.headers['X-Sogou-Tag'] = calc_sogou_hash(t, self.headers['Host'])
self.headers['X-Sogou-Timestamp'] = t
self.s.send(str(self.headers).encode('ascii') + b'\r\n')
# Send Post data
if(self.command=='POST'):
self.s.send(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.s, method=self.command, buffering=True)
response.begin()
# Reply to the browser
status = 'HTTP/1.1 ' + str(response.status) + ' ' + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
h = ''
for hh, vv in response.getheaders():
if hh.upper()!='TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h.encode('ascii') + b'\r\n')
while True:
response_data = response.read(8192)
if(len(response_data) == 0):
break
self.wfile.write(response_data)
except socket.error:
print('socket error for ' + self.requestline)
def sogouProxy(self):
if self.remote is None or self.lastHost != self.headers["Host"]:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.settimeout(RemoteTimeout)
self.remote.connect((proxy_host, proxy_port))
self.remote.sendall(self.requestline.encode('ascii').replace('baike.baidu.com', '220.181.111.247') + b"\r\n")
# Add Sogou Verification Tags
self.headers["X-Sogou-Auth"] = x_sogou_auth
t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
self.headers["X-Sogou-Tag"] = calc_sogou_hash(t, self.headers['Host'])
self.headers["X-Sogou-Timestamp"] = t
headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
# Send Post data
if self.command == 'POST':
self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
hlist = []
for line in response.msg.headers: # Fixed multiple values of a same name
if 'TRANSFER-ENCODING' not in line.upper():
hlist.append(line)
self.wfile.write("".join(hlist) + b'\r\n')
if self.command == "CONNECT" and response.status == 200:
return self.transfer(self.remote, self.connection)
else:
while True:
response_data = response.read(BufferSize)
if not response_data: break
self.wfile.write(response_data)
def sogouProxy(self):
if self.remote is None or self.lastHost != self.headers["Host"]:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.settimeout(RemoteTimeout)
self.remote.connect((proxy_host, proxy_port))
self.remote.sendall(self.requestline.encode('ascii') + b"\r\n")
# Add Sogou Verification Tags
self.headers["X-Sogou-Auth"] = x_sogou_auth
t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
self.headers["X-Sogou-Tag"] = calc_sogou_hash(t, self.headers['Host'])
self.headers["X-Sogou-Timestamp"] = t
headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
self.remote.sendall(headerstr.encode('ascii') + b"\r\n")
# Send Post data
if self.command == 'POST':
self.remote.sendall(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self.remote, method=self.command)
response.begin()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
hlist = []
for line in response.msg.headers: # Fixed multiple values of a same name
if 'TRANSFER-ENCODING' not in line.upper():
hlist.append(line)
self.wfile.write("".join(hlist) + b'\r\n')
if self.command == "CONNECT" and response.status == 200:
return self.transfer(self.remote, self.connection)
else:
while True:
response_data = response.read(BufferSize)
if not response_data: break
self.wfile.write(response_data)
def testPipelining(self):
# Tests the use of several requests issued at once.
s = ("GET / HTTP/1.0\r\n"
"Connection: %s\r\n"
"Content-Length: %d\r\n"
"\r\n"
"%s")
to_send = ''
count = 25
for n in range(count):
body = "Response #%d\r\n" % (n + 1)
if n + 1 < count:
conn = 'keep-alive'
else:
conn = 'close'
to_send += s % (conn, len(body), body)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(to_send)
for n in range(count):
expect_body = "Response #%d\r\n" % (n + 1)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(response_body))
self.failUnlessEqual(response_body, expect_body)
def issue_row(raw_row):
issue_row = {}
for column in COLUMN_HEADERS:
column_data_raw = raw_row.findtext(column)
if column_data_raw:
if column in ['issueDetail', 'issueBackground', 'remediationBackground']:
issue_row[column] = htmltext(column_data_raw)
else:
issue_row[column] = column_data_raw
if len(issue_row[column]) > 32000:
issue_row[column] = "".join(issue_row[column][:32000], " [Text Cut Due To Length]")
request = raw_row.findtext('./requestresponse/request')
if request:
parsed_request = HTTPRequest(binascii.a2b_base64(request))
formatted_request_a = "command : {}\nuri : {}\nrequest_version : {}".format(parsed_request.command, parsed_request.path, parsed_request.request_version)
formatted_request_b = "\n".join("{}: {}".format(header, parsed_request.headers[header]) for header in parsed_request.headers.keys())
issue_row['requestHeaders'] = "{}\n{}".format(formatted_request_a, formatted_request_b)
response = raw_row.findtext('./requestresponse/response')
if response:
parsed_response = HTTPResponse(FakeSocket(binascii.a2b_base64(response)))
parsed_response.begin()
formatted_response = "\n".join(["{} : {}".format(header_item[0], header_item[1]) for header_item in parsed_response.getheaders()])
issue_row['responseHeaders'] = formatted_response
return issue_row
def open_image2(url, buff_size=4096, connect_timeout=5, read_timeout=4,
full_read_timeout=30, logger=logging):
t = urlparse.urlparse(url)
path = t.path or '/'
if len(t.netloc.split(':')) == 2:
HOST, PORT = t.netloc.split(':')
else:
HOST, PORT = t.netloc, 80
UA = ('Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36'
' (KHTML, like Gecko) Chrome/54.0.2840.87 Safari/537.36')
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(connect_timeout)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
s.connect((HOST, PORT))
except Exception as e:
logger.exception(e)
return None
s.send("GET %s HTTP/1.1\r\n" % path)
s.send("Host:%s\r\n" % HOST)
s.send("Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n")
s.send("User-Agent:%s\r\n" % UA)
s.send('\r\n')
data = []
begin = time.time()
while 1:
if time.time() - begin > full_read_timeout:
logger.error('reach full_read_timeout')
data = []
break
try:
msg = s.recv(buff_size)
#print len(msg), len(data), time.time() - begin
if len(msg) == 0:
break
data.append(msg)
except socket.timeout as e:
if e.args[0] == 'timed out':
break
else:
logger.error('open image [%s] socket timeout: %s' % (url, str(e)))
data = []
break
except socket.error as e:
logger.error('open image [%s] socket error: %s' % (url, str(e)))
data = []
break
s.shutdown(1)
s.close()
data = ''.join(data)
if data:
resp = HTTPResponse(FakeSocket(data))
resp.begin()
if resp.status == 200:
return resp.read(len(data))
return None
def __init__(self, response):
r = HTTPResponse(self._FakeSocket(response))
r.begin()
self.location = r.getheader("location")
self.usn = r.getheader("usn")
self.st = r.getheader("st")
self.cache = r.getheader("cache-control").split("=")[1]
try: r.close()
except: pass
def httpparse(fp):
try:
contents = fp.read()
except AttributeError:
contents = fp
socket = FakeSocket(contents)
response = HTTPResponse(socket)
response.begin()
return response
def get(self, request):
# Format and pass along the initial request
request = self.format_request(request)
self.outsocket.sendall(request)
# Wait for request
while (True):
r, w, x = select.select([self.insocket, self.outsocket], [], [], 0)
if self.insocket in r:
request = self.insocket.recv(buff_size)
if request == "":
print "Got end message from browser"
self.kill()
else:
try:
request = self.format_request(request)
except MismatchedHost:
print "Host changed. Getting new end socket"
self.insocket.sendall("")
self.kill()
self.outsocket.sendall(request)
if self.outsocket in r:
httpRes = HTTPResponse(self.outsocket)
response = ""
try:
httpRes.begin()
headers = str(httpRes.msg)
print headers
content = httpRes.read()
# TODO: Move below to format_response
# Fix chunked header
if headers.find("Transfer-Encoding: chunked") != -1:
headers = headers.replace(
"Transfer-Encoding: chunked\r\n", "")
headers += "Content Length: " + str(
len(content)) + "\r\n"
if httpRes.version == 10:
response += "HTTP/1.0 "
elif httpRes.version == 11:
response += "HTTP/1.1 "
response += str(httpRes.status) + " " + str(
httpRes.reason) + nl
response += headers + nl
response += content
print response
except BadStatusLine:
self.kill()
self.insocket.sendall(response)
self.kill()
if self.killed:
self.end()
return
def init(self, response_text):
class FakeSocket:
def __init__(self, response_str):
self._file = StringIO(response_str)
def makefile(self, *args, **kwargs):
return self._file
source = FakeSocket(response_text)
HTTPResponse.__init__(self, source)
self.begin()
def filter_malware(response_str):
global apikey
source = HTTPObject(response_str)
http_response = HTTPResponse(source)
http_response.begin()
url = 'https://www.virustotal.com/vtapi/v2/file/report'
content = http_response.read(len(response_str))
hash = hashlib.md5(content).hexdigest()
params = {'apikey': apikey, 'resource': hash}
response = requests.get(url, params=params)
return response
def get_response(self):
"""Return the parse XML response.
If no data was read, raises EOFError."""
self.read_buffer.seek(0)
if self.amount_read == 0:
raise EOFError("Read zero bytes")
self.logger.debug("Processing response of %d bytes" % self.amount_read)
response = HTTPResponse(self.read_buffer)
response.begin() # Process the response
return response
def _parse_http(self, text):
# if the response text starts with a 302, skip to the next non-302 header
if re.match(r'^HTTP/.*?\s302 Found', text):
m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
if not m:
raise Exception("Unrecognized response: %s" % text)
else:
text = m.group(1)
socket = self.FakeSocket(text)
response = HTTPResponse(socket)
response.begin()
return response
def parse_response_header(seg):
source = FakeSocket(seg)
response = HTTPResponse(source)
response.begin()
http_v = {10: 'HTTP/1.0', 11: 'HTTP/1.1'}
res_data = {}
res_data['status'] = response.status
res_data['response_version'] = http_v[response.version]
data = response.getheaders()
for h in data:
res_data[str(h[0])] = h[1]
return res_data
def _parse_http(self, text):
# if the response text starts with a 302, skip to the next non-302 header
if re.match(r'^HTTP/.*?\s302 Found', text):
m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
if not m:
raise Exception("Unrecognized response: %s" % text)
else:
text = m.group(1)
socket = self.FakeSocket(text)
response = HTTPResponse(socket)
response.begin()
return response
def proccessHttpResponse(self, burpHttpReqResp):
""" Processes only the HTTP repsonses with a CSP header """
byteResponse = burpHttpReqResp.getResponse()
httpSocket = HttpDummySocket(bytearray(byteResponse))
response = HTTPResponse(httpSocket)
response.begin()
issues = []
for header in response.getheaders():
if header[0].lower() in ContentSecurityPolicy.HEADERS:
findings = self.parseContentSecurityPolicy(header, burpHttpReqResp)
issues.extend(findings)
return issues
def proccessHttpResponse(self, burpHttpReqResp):
""" Processes only the HTTP repsonses with a CSP header """
byteResponse = burpHttpReqResp.getResponse()
httpSocket = HttpDummySocket(bytearray(byteResponse))
response = HTTPResponse(httpSocket)
response.begin()
issues = []
for header in response.getheaders():
if header[0].lower() in ContentSecurityPolicy.HEADERS:
findings = self.parseContentSecurityPolicy(header, burpHttpReqResp)
issues.extend(findings)
return issues
def decompress_payload(payload):
try:
source = FakeSocket(payload)
res = HTTPResponse(source)
res.begin()
result = zlib.decompress(res.read(), 16 + zlib.MAX_WBITS)
except Exception as e:
result = payload
# try:
# result = '.'.join(str(ord(c)) for c in payload)
# except:
# result = payload
return result
def begin(self):
HTTPResponse.begin(self)
# To void the broken logic at the end of begin, because
# but the connection may stay explicitly open with a connect
# of for some other reason
self.will_close = self._check_close()
# it is sensible to assume that after a connect if
# 200 is returned, the connection will not close
# even if issued as 1.0 and no specific connection
# header came back - a binary connection is now open
if self._method == 'CONNECT' and self.status == 200:
self.will_close = 0
def getBody(msg):
''' Get the entity body section of the http message '''
class FakeSocket():
def __init__(self, msg):
self.file = StringIO(msg)
def makefile(self, *args, **kwargs):
return self.file
source = FakeSocket(msg)
rsp = HTTPResponse(source)
rsp.begin()
return rsp.read(len(msg))
def begin(self):
HTTPResponse.begin(self)
# To void the broken logic at the end of begin, because
# but the connection may stay explicitly open with a connect
# of for some other reason
self.will_close = self._check_close()
# it is sensible to assume that after a connect if
# 200 is returned, the connection will not close
# even if issued as 1.0 and no specific connection
# header came back - a binary connection is now open
if self._method == 'CONNECT' and self.status == 200:
self.will_close = 0
def setUp(self):
from httplib import HTTPResponse
with Stub() as HTTPResponse:
HTTPResponse.status >> 200
HTTPResponse.read() >> '{"id": "THE-PRECIOUS-GUID", "title": "example title", "subtitle": "example title - subtitle", "description": "a description", "user": "******", "tags": [], "created_at": 1329767353.0, "source": "http://www.example.com/example.csv", "link": "http://www.junar.com/datastreams/some-url"}'
with Stub() as conn:
from httplib import HTTPConnection
conn = HTTPConnection("api.junar.com")
conn.request("POST", "/datastreams/publish", self.params, self.headers)
conn.getresponse() >> HTTPResponse
def get(self, request):
# Format and pass along the initial request
request = self.format_request(request)
self.outsocket.sendall(request)
# Wait for request
while(True):
r,w,x = select.select([self.insocket, self.outsocket],[],[],0)
if self.insocket in r:
request = self.insocket.recv(buff_size)
if request == "":
print "Got end message from browser"
self.kill()
else:
try:
request = self.format_request(request)
except MismatchedHost:
print "Host changed. Getting new end socket"
self.insocket.sendall("")
self.kill()
self.outsocket.sendall(request)
if self.outsocket in r:
httpRes = HTTPResponse(self.outsocket)
response = ""
try:
httpRes.begin()
headers = str(httpRes.msg)
content = httpRes.read()
if headers.find("Transfer-Encoding: chunked") != -1:
headers = headers.replace("Transfer-Encoding: chunked\r\n", "")
headers += "Content Length: " + str(len(content)) + "\r\n"
if httpRes.version == 10:
response += "HTTP/1.0 "
elif httpRes.version == 11:
response += "HTTP/1.1 "
response += str(httpRes.status) + " " + str(httpRes.reason) + nl
response += headers + nl
response += content
#print response
except BadStatusLine:
self.kill()
self.insocket.sendall(response)
self.kill()
#'''
if self.killed:
self.end()
return
def send_and_recv(self):
try:
# because www.dream-pro.info is tlanslated to 127.0.0.1 using hosts' entry,
# send message to www.dream-pro.info with socket.socket to make
# http connection
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((WWW_DREAM_PRO_INFO, 80))
sock.sendall(str(self))
except:
print 'SocketError'
return
res = HTTPResponse(sock)
res.begin()
res_body = res.read()
res.close()
if 'transfer-encoding' in res.msg:
# httplib.HTTPResponse automatically concatenate chunked response
# but do not delete 'transfer-encoding' header
# so the header must be deleted
res.msg.__delitem__('transfer-encoding')
compmeth = res.msg.getheader('content-encoding', '').lower()
if compmeth and compmeth.find('identity') != 0:
# response body is compressed with some method
offset = 0
if compmeth.find('gzip') != -1:
# if body is gziped, header offset value is 47
# if not, offset value is 0
# this server does not support sdch...
offset += 47
res_body = decompress(res_body, offset)
res.msg['content-encoding'] = 'identity'
return res, res_body
def testKeepaliveHttp10(self):
# Handling of Keep-Alive within HTTP 1.0
data = "Default: Don't keep me alive"
s = ("GET / HTTP/1.0\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
connection = response.getheader('Connection', '')
# We sent no Connection: Keep-Alive header
# Connection: close (or no header) is default.
self.failUnless(connection != 'Keep-Alive')
# If header Connection: Keep-Alive is explicitly sent,
# we want to keept the connection open, we also need to return
# the corresponding header
data = "Keep me alive"
s = ("GET / HTTP/1.0\n"
"Connection: Keep-Alive\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
connection = response.getheader('Connection', '')
self.failUnlessEqual(connection, 'Keep-Alive')
def sendAndReceive(self):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((ORIGINAL_IP, 80))
sock.sendall(str(self))
except:
return GlobalTools.FailedHTTPResponse(), ''
res = HTTPResponse(sock)
res.begin()
res_body = res.read()
res.close()
if 'transfer-encoding' in res.msg:
# httplib.HTTPResponse automatically concatenate chunked response
# but do not delete 'transfer-encoding' header
# so the header must be deleted
res.msg.__delitem__('transfer-encoding')
compmeth = res.msg.getheader('content-encoding', '').lower()
if compmeth and compmeth.find('identity') != 0:
# response body is compressed with some method
offset = 0
if compmeth.find('gzip') != -1:
# if body is gziped, header offset value is 47
# if not, offset value is 0
# this server does not support sdch...
offset += 47
res_body = decompress(res_body, offset)
res.msg['content-encoding'] = 'identity'
return res, res_body
def validate_for_regular_http_api_response(self,request_verb):
try:
s = self.get_socket(self.host,self.current_port)
request_string = self.get_request_templates(request_verb)
#print request_string
s.sendall(request_string)
data = HTTPResponse(s)
#data = (s.recv(self.total_buffer))
data.begin()
s.shutdown(1)
s.close()
print TEST_CASE_RESULT_TEMPLATE % ("RD Service HTTP " +request_verb + " disabled verification " , TEST_CASE_RESULT_FAILURE)
except socket.error,exc:
print TEST_CASE_RESULT_TEMPLATE % ("RD Service HTTP " +request_verb + " disabled verification ", TEST_CASE_RESULT_PASS)
def _do_handshake(self, url, version=6, origin=None, cookie=None):
"""http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-06"""
urlParts = urlparse.urlparse(url)
key = b64encode(urandom(16))
request = "GET %s HTTP/1.1\r\n" \
"Host: %s\r\n" \
"Upgrade: WebSocket\r\n" \
"Connection: Upgrade\r\n" \
"Sec-WebSocket-Key: %s\r\n" \
"Sec-WebSocket-Version: %d\r\n" \
% (urlParts.path, self.host, key, version)
if origin is not None:
request += "Sec-WebSocket-Origin: %s\r\n" % (origin,)
if cookie is not None:
request += cookie.output(header="Cookie:") + "\r\n"
# Authenticate if required
if self.username is not None:
request += "Authorization: Basic %s\r\n" \
% b64encode('%s:%s' % (self.username, self.password or ''))
# Finish request
request += "\r\n"
if _debug:
print >> sys.stderr, '\x1B[D\x1B[31m%s\x1B[m' % (request,),
self.socket.sendall(request)
response = HTTPResponse(self.socket)
response.begin()
if _debug:
print >> sys.stderr, '\x1B[D\x1B[34m%s' % ({9: 'HTTP/0.9', 10: 'HTTP/1.0', 11: 'HTTP/1.1'}[response.version],), response.status, response.reason
print >> sys.stderr, '%s\x1B[m' % (response.msg,)
if response.status != 101:
self.socket.close()
raise RuntimeError("WebSocket upgrade failed: %d %s" % (response.status, response.reason))
expected = sha1(key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").digest()
assert len(expected) == 20
expected = b64encode(expected)
if 'Sec-WebSocket-Accept' not in response.msg:
raise RuntimeError, "Expected WebSocket header not present: Sec-WebSocket-Accept"
if response.msg['Sec-WebSocket-Accept'].strip() != expected:
raise RuntimeError, "Invalid WebSocket accept returned: %s %s %s" % (key, expected, response.msg['Sec-WebSocket-Accept'])
def connect_rd_services(self,port): try: s = self.get_socket(self.host,port) request_string = self.get_request_templates() #print request_string s.sendall(request_string) data = HTTPResponse(s) #data = (s.recv(self.total_buffer)) data.begin() s.shutdown(1) s.close() return data except socket.error,exc: print "Error Connecting to Port " + str(port) return 0
def get(addr, path, debug=False):
client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
client.connect(addr)
client.send("GET {} HTTP/1.0\r\n\r\n".format(path))
resp_str = client.recv(65536)
source = FakeSocket(resp_str)
resp = HTTPResponse(source)
resp.begin()
if resp.status == 200:
text = resp.read(len(resp_str))
data = json.loads(text)
if debug:
log.debug(data)
return data
return {}
def get(addr, path, debug=False):
client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
client.connect(addr)
client.send("GET {} HTTP/1.0\r\n\r\n".format(path))
resp_str = client.recv(65536)
source = FakeSocket(resp_str)
resp = HTTPResponse(source)
resp.begin()
if resp.status == 200:
text = resp.read(len(resp_str))
data = json.loads(text)
if debug:
log.debug(data)
return data
return {}
def _parse_http(self, text):
# if the response text starts with a 302, skip to the next non-302 header
if re.match(r'^HTTP/.*?\s302 Found', text):
m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
if not m:
raise Exception("Unrecognized response: %s" % text)
else:
text = m.group(1)
# if the response text starts with a "200 Connection established" but continues with a 201,
# skip the 200 header. This happens when using a proxy.
#
# e.g. HTTP/1.1 200 Connection established
# Via: 1.1 proxy
# Connection: Keep-Alive
# Proxy-Connection: Keep-Alive
#
# HTTP/1.1 201 Created
# Server: GitHub.com
# ...
# Status: 201 Created
# ...
if re.match(r'^HTTP/.*?\s200 Connection established', text):
m = re.search(
r'(HTTP/\d+\.\d+\s(?!200 Connection established).*$)', text,
re.S)
if not m:
raise Exception("Unrecognized response: %s" % text)
else:
text = m.group(1)
# remove Transfer-Encoding: chunked header, as it causes reading the response to fail
# first do a quick check for it, so we can avoid doing the expensive negative-lookbehind
# regex if we don't need it
if "Transfer-Encoding: chunked" in text:
# we do the negative-lookbehind to make sure we only strip the Transfer-Encoding
# string in the header
text = re.sub(r'(?<!\r\n\r\n).*?Transfer-Encoding: chunked\r\n',
'',
text,
count=1)
logger.debug("CurlSession - getting socket from %s" % text)
socket = self.FakeSocket(text)
response = HTTPResponse(socket)
response.begin()
return response
class PassThroughProxyDownloader(DownloaderBase):
def __init__(self, user_agent, proxy):
self.user_agent = user_agent
web_proxy_split = proxy.split(':')
self.proxy_ip = web_proxy_split[0]
self.proxy_port = int(web_proxy_split[1])
def download(self, url, referrer=None, if_modified_since=None, if_none_match=None):
request = [
"GET %s HTTP/1.1" % url,
"User-Agent: %s" % self.user_agent,
"Accept-Encoding: gzip, deflate, compress",
"Accept: */*"
"Host: %s" % urlparse(url).hostname,
"Connection: Close" # this may not be necessary
]
for header, value in CommonHeaders.iteritems():
request.append("%s: %s" % (header, value))
for header, value in DownloaderBase.construct_headers(referrer, if_modified_since, if_none_match).iteritems():
request.append("%s: %s" % (header, value))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((self.proxy_ip, self.proxy_port))
except socket.error, ex:
raise ProxyDownException(ex)
s.send("\r\n".join(request) + '\r\n\r\n')
r = HTTPResponse(s, strict=False, method='GET', buffering=True)
r.begin()
resp = urllib3Response.from_httplib(r, decode_content=False)
s.close()
r.close()
response = requests.Response()
# Fallback to None if there's no status_code, for whatever reason.
response.status_code = getattr(resp, 'status', None)
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
response.encoding = get_encoding_from_headers(response.headers)
response._content = resp.data
response.raw = resp
response.reason = response.raw.reason
response.url = url
# don't care about cookies right now
#extract_cookies_to_jar(response.cookies, req, resp)
# don't worry about the requests' Request object right now until it is needed
#response.request = req
response.request = None
return response
def import_incidents(request, source):
'''
Master incident import view. Used for the initial population of the dispatch database.
'''
if source == 'twitter':
get_twitter_incidents(request, TWITTER_USERNAME)
return HTTPResponse(status=200)
elif source == 'email':
get_email_incidents(request, EMAIL_USERNAME)
return HTTPResponse(status=200)
else:
raise ValueError(
"Invalid dispatch source. Please specify twitter, email or cad.")
def get_json(self, response):
res_bytes = response.getResponse() # byte[]
if not res_bytes:
print "[!] Failed to get json for Response. Response has not bytes"
return {}
res_str = str(helpers.bytesToString(res_bytes))
source = FakeSocket(res_str)
res_obj = HTTPResponse(source)
res_obj.begin()
# print "Reading response object"
res_text = res_obj.read()
try:
res_data = json.loads(res_text)
except Exception as e:
print "[!] Failed to convert to json: " + str(res_text)
res_data = {}
return res_data
def getheaders(self):
"""Return all headers from the response
This gives the ability to access all HTTP headers of a discovery response.
:return: dict[str, str]
"""
return HTTPResponse.getheaders(self)
def http_has_valid_content_length(all_data):
"""
Gets HTTP data, returns:
1. True if it has a header "content_length" and its value <= MAX_HTTP_CONTENT_LENGTH,
2. False otherwise (includes the case were all_data doesn't contain this header)
NOTE: USE THIS ONLY ON HTTP DATA!
"""
source = FakeSocket(all_data)
response = HTTPResponse(source)
response.begin()
content_len_value = int(response.getheader('content-length', -1))
if content_len_value == -1 or content_len_value > MAX_HTTP_CONTENT_LENGTH:
print("Invalid (or missing) content-length header.")
return False
return True
def getheaders(self):
"""Return all headers from the response
This gives the ability to access all HTTP headers of a discovery response.
:return: dict[str, str]
"""
return HTTPResponse.getheaders(self)
def _parse_ws_response(self):
http_response = self.__ws.recv()
response = HTTPResponse(FakeSocket(http_response))
response.begin()
while response.length > len(http_response):
http_response += self.__ws.recv()
response = HTTPResponse(FakeSocket(http_response))
self.__transport.verbose = 0
response.begin()
return self.__transport.parse_response(response)
def _get_response(self):
# Parse response
h = HTTPResponse(self._proxy_sock)
h.begin()
res_d = {
'status':
h.status,
'reason':
h.reason,
'req_version':
self.request_version,
'msg':
h.msg,
'trans_enc': (h.msg['Transfer-Encoding']
if 'Transfer-Encoding' in h.msg else '[NO-ENCODING]')
}
# Get rid of the pesky header
del h.msg['Transfer-Encoding']
res = '%(req_version)s %(status)s %(reason)s\r\n%(msg)s\r\n' % res_d
res_d['data'] = h.read()
res_d['time'] = time()
res += res_d['data']
# Let's close off the remote end
h.close()
self._proxy_sock.close()
# Relay the message
self.request.sendall(self.mitm_response(res))
print res
return res_d
def __init__(self, payload):
r = HTTPResponse(self._fakeSocket(payload))
r.begin()
self.st = r.getheader("st") or None
self.usn = r.getheader("usn") or None
self.server = r.getheader("server") or None
self.location = r.getheader("location") or None
def send_and_recv(self):
try:
# because www.dream-pro.info is tlanslated to 127.0.0.1 using hosts' entry,
# send message to www.dream-pro.info with socket.socket to make
# http connection
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((WWW_DREAM_PRO_INFO,80))
sock.sendall(str(self))
except:
print 'SocketError'
return
res = HTTPResponse(sock)
res.begin()
res_body = res.read()
res.close()
if 'transfer-encoding' in res.msg:
# httplib.HTTPResponse automatically concatenate chunked response
# but do not delete 'transfer-encoding' header
# so the header must be deleted
res.msg.__delitem__('transfer-encoding')
compmeth = res.msg.getheader('content-encoding','').lower()
if compmeth and compmeth.find('identity') != 0 :
# response body is compressed with some method
offset = 0
if compmeth.find('gzip') != -1:
# if body is gziped, header offset value is 47
# if not, offset value is 0
# this server does not support sdch...
offset += 47
res_body = decompress(res_body,offset)
res.msg['content-encoding'] = 'identity'
return res, res_body
def parse_response(response_text):
"""
Given an HTTP response line and headers, return a requests.Response object.
"""
class FakeSocket():
def __init__(self, response_str):
self._file = StringIO(response_str)
def makefile(self, *args, **kwargs):
return self._file
source = FakeSocket(response_text)
response = HTTPResponse(source)
response.begin()
requests_response = requests.Response()
requests_response.status_code = response.status
requests_response.headers = CaseInsensitiveDict(response.getheaders())
return requests_response
def SOCKET_SEND(http_packet):
# SEND REQUEST
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssl_s = ssl.wrap_socket(s)
ssl_s.connect((host_address, https_port))
if SHOW_HTTP_REQUESTS: print ('--- Sending ---\r\n' + http_packet + '\r\n----')
ssl_s.send(http_packet)
# GET RESPONSE
response = ssl_s.recv(1024)
ssl_s.close()
if SHOW_HTTP_REQUESTS: print ('--- Response --- \r\n' + str(response) + '\r\n---')
#PARSE REPONSE
fake_socket_response = FakeSocket(response)
parsed_response = HTTPResponse(fake_socket_response)
parsed_response.begin()
return parsed_response
def _read_resp(self):
# Let httplib handle the response part.
from httplib import HTTPResponse
from urllib import addinfourl
resp = HTTPResponse(self.sock, strict=True, method=self.method)
resp.begin()
if resp.will_close:
self.sock.close()
else:
# TODO Should make something of this, I suppose.
self._active_resp = resp
# Courtesy of urllib2, weird stuff going on here.
resp.recv = resp.read
fp = socket._fileobject(resp, close=True)
rv = addinfourl(fp, resp.msg, self.get_full_url())
rv.code = resp.status
rv.msg = resp.reason
return rv
def testWithoutCRLF(self):
# Tests the use of just newlines rather than CR/LFs.
data = "Echo\nthis\r\nplease"
s = ("GET / HTTP/1.0\n"
"Connection: close\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(data))
self.failUnlessEqual(response_body, data)
def _parse_http(self, text):
# if the response text starts with a 302, skip to the next non-302 header
if re.match(r'^HTTP/.*?\s302 Found', text):
m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
if not m:
raise Exception("Unrecognized response: %s" % text)
else:
text = m.group(1)
# remove Transfer-Encoding: chunked header, as it causes reading the response to fail
# first do a quick check for it, so we can avoid doing the expensive negative-lookbehind
# regex if we don't need it
if "Transfer-Encoding: chunked" in text:
# we do the negative-lookbehind to make sure we only strip the Transfer-Encoding
# string in the header
text = re.sub(r'(?<!\r\n\r\n).*?Transfer-Encoding: chunked\r\n', '', text, count=1)
socket = self.FakeSocket(text)
response = HTTPResponse(socket)
response.begin()
return response
def _parse_http(self, text):
# if the response text starts with a 302, skip to the next non-302 header
if re.match(r'^HTTP/.*?\s302 Found', text):
m = re.search(r'(HTTP/\d+\.\d+\s(?!302 Found).*$)', text, re.S)
if not m:
raise Exception("Unrecognized response: %s" % text)
else:
text = m.group(1)
# if the response text starts with a "200 Connection established" but continues with a 201,
# skip the 200 header. This happens when using a proxy.
#
# e.g. HTTP/1.1 200 Connection established
# Via: 1.1 proxy
# Connection: Keep-Alive
# Proxy-Connection: Keep-Alive
#
# HTTP/1.1 201 Created
# Server: GitHub.com
# ...
# Status: 201 Created
# ...
if re.match(r'^HTTP/.*?\s200 Connection established', text):
m = re.search(r'(HTTP/\d+\.\d+\s(?!200 Connection established).*$)', text, re.S)
if not m:
raise Exception("Unrecognized response: %s" % text)
else:
text = m.group(1)
# remove Transfer-Encoding: chunked header, as it causes reading the response to fail
# first do a quick check for it, so we can avoid doing the expensive negative-lookbehind
# regex if we don't need it
if "Transfer-Encoding: chunked" in text:
# we do the negative-lookbehind to make sure we only strip the Transfer-Encoding
# string in the header
text = re.sub(r'(?<!\r\n\r\n).*?Transfer-Encoding: chunked\r\n', '', text, count=1)
logger.debug("CurlSession - getting socket from %s" % text)
socket = self.FakeSocket(text)
response = HTTPResponse(socket)
response.begin()
return response
def __init__(self, response_str):
source = FakeSocket(response_str)
HTTPResponse.__init__(self, source)
self.begin()
# Request
# error_code
# command GET/POST
# path PATH
# request_version "HTTP/1.0"
# headers "Headers"
# headers.keys()
# headers['host']
# Response
# status CODE
# getheader('Content-Length')
# read(len(response_str))
def parse_http_response(sock):
try:
# H4ck to standardize the API between sockets and SSLConnection objects
response = sock.read(4096)
except AttributeError:
response = sock.recv(4096)
if 'HTTP/' not in response:
# Try to get the rest of the response
try:
response += sock.read(4096)
except AttributeError:
response += sock.recv(4096)
fake_sock = FakeSocket(response)
response = HTTPResponse(fake_sock)
response.begin()
return response
def parse_response(response_text):
"""
Given an HTTP response line and headers, return a requests.Response object.
"""
class FakeSocket():
def __init__(self, response_str):
self._file = StringIO(response_str)
def makefile(self, *args, **kwargs):
return self._file
source = FakeSocket(response_text)
response = HTTPResponse(source)
response.begin()
requests_response = requests.Response()
requests_response.status_code = response.status
headers = CaseInsensitiveDict(response.getheaders())
# Reset headers['x-robots-tag'], so that we can handle the
# possibilility that multiple x-robots directives might be included
# https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag
# e.g.
# HTTP/1.1 200 OK
# Date: Tue, 25 May 2010 21:42:43 GMT
# (...)
# X-Robots-Tag: googlebot: nofollow
# X-Robots-Tag: otherbot: noindex, nofollow
# (...)
# Join with a semi-colon, not a comma, so that multiple agents can
# be recovered. As of 12/14/16, there doesn't appear to be any spec
# describing how to do this properly (since commas don't work).
# Since parsed response headers aren't archived, this convenience is
# fine. However, it's worth keeping track of the situation.
robots_directives = []
for directive in response.msg.getallmatchingheaders('x-robots-tag'):
robots_directives.append(directive.split(": ", 1)[1].replace("\n", "").replace("\r", ""))
headers['x-robots-tag'] = ";".join(robots_directives)
requests_response.headers = headers
return requests_response
def process_record(self, record):
content = None
try:
payload = record.payload.read()
s = FakeSocket(payload)
response = HTTPResponse(s)
response.begin()
status_code = response.status
if status_code != 200:
return
content_type = response.getheader('Content-Type', '')
if 'text/html' not in content_type:
return
headers = response.getheaders()
content = response.read(len(payload))
except Exception:
self.increment_counter('errors', 'process_record', 1)
logging.error('Error processing record: {}', traceback.format_exc())
return
if content is not None:
content = content.strip()
if not content:
return
for item in self.process_content(record.url, headers, content):
yield item