def testKeepaliveHttp10(self):
# Handling of Keep-Alive within HTTP 1.0
data = "Default: Don't keep me alive"
s = ("GET / HTTP/1.0\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
connection = response.getheader('Connection', '')
# We sent no Connection: Keep-Alive header
# Connection: close (or no header) is default.
self.failUnless(connection != 'Keep-Alive')
# If header Connection: Keep-Alive is explicitly sent,
# we want to keept the connection open, we also need to return
# the corresponding header
data = "Keep me alive"
s = ("GET / HTTP/1.0\n"
"Connection: Keep-Alive\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
connection = response.getheader('Connection', '')
self.failUnlessEqual(connection, 'Keep-Alive')
def __init__(self, payload):
r = HTTPResponse(self._fakeSocket(payload))
r.begin()
self.st = r.getheader("st") or None
self.usn = r.getheader("usn") or None
self.server = r.getheader("server") or None
self.location = r.getheader("location") or None
def parse_response(self, response):
""" Check if the response is valid to stored in cache """
global verbose2
try:
source = FakeSocket(response)
parsed_response = HTTPResponse(source)
parsed_response.begin()
except Exception as e:
print("Error in parsing response. " + str(e))
return 0
sc = parsed_response.status # status-code
try:
cc = parsed_response.getheader("Cache-Control").split(
',') # cache-control
except:
cc = []
pragma = parsed_response.getheader("Pragma")
if verbose2:
print("sc: " + str(sc) + ", pragma: " + str(pragma) +
", cache-control: " + str(';'.join(cc)))
if sc == 302 or sc == 301 or sc == 200 or sc == 304:
if 'no-cache' in cc or 'private' in cc or 'no-store' in cc or pragma == 'no-cache':
return 0
else:
return 1
else:
return 0
def __init__(self, response):
r = HTTPResponse(self._FakeSocket(response))
r.begin()
self.location = r.getheader("location")
self.usn = r.getheader("usn")
self.st = r.getheader("st")
self.cache = r.getheader("cache-control").split("=")[1]
try: r.close()
except: pass
def __init__(self, response_text):
self.fp = FakeSocket(response_text)
res = HTTPR(self.fp)
res.begin()
headers = res.getheaders()
for header in headers:
self.headers[header[0]] = header[1]
self.length = res.getheader('Content-Length')
self.chunked = res.getheader('Transfer-Encoding')
def testKeepaliveHttp11(self):
# Handling of Keep-Alive within HTTP 1.1
# All connections are kept alive, unless stated otherwise
data = "Default: Keep me alive"
s = ("GET / HTTP/1.1\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
self.failUnless(response.getheader('connection') != 'close')
# Explicitly set keep-alive
data = "Default: Keep me alive"
s = ("GET / HTTP/1.1\n"
"Connection: keep-alive\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
self.failUnless(response.getheader('connection') != 'close')
# no idea why the test publisher handles this request incorrectly
# it would be less typing in the test :)
# h = HTTPConnection(LOCALHOST, self.port)
# h.request("GET", "/")
# response = h.getresponse()
# self.failUnlessEqual(int(response.status), 200)
# self.failUnless(response.getheader('connection') != 'close')
# specifying Connection: close explicitly
data = "Don't keep me alive"
s = ("GET / HTTP/1.1\n"
"Connection: close\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
self.failUnlessEqual(response.getheader('connection'), 'close')
def testPipelining(self):
# Tests the use of several requests issued at once.
s = ("GET / HTTP/1.0\r\n"
"Connection: %s\r\n"
"Content-Length: %d\r\n"
"\r\n"
"%s")
to_send = ''
count = 25
for n in range(count):
body = "Response #%d\r\n" % (n + 1)
if n + 1 < count:
conn = 'keep-alive'
else:
conn = 'close'
to_send += s % (conn, len(body), body)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(to_send)
for n in range(count):
expect_body = "Response #%d\r\n" % (n + 1)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(response_body))
self.failUnlessEqual(response_body, expect_body)
def testPipelining(self):
# Tests the use of several requests issued at once.
s = ("GET / HTTP/1.0\r\n"
"Connection: %s\r\n"
"Content-Length: %d\r\n"
"\r\n"
"%s")
to_send = ''
count = 25
for n in range(count):
body = "Response #%d\r\n" % (n + 1)
if n + 1 < count:
conn = 'keep-alive'
else:
conn = 'close'
to_send += s % (conn, len(body), body)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(to_send)
for n in range(count):
expect_body = "Response #%d\r\n" % (n + 1)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(response_body))
self.failUnlessEqual(response_body, expect_body)
def process_http(raw):
headers, body = raw.split(BODY_SEP, 1)
if headers.startswith('CONNECT'):
return None
if not len(body):
return {
'headers': headers,
'body': None
}
source = FakeSocket(raw)
response = HTTPResponse(source)
response.begin()
body = None
if response.getheader('Content-Encoding') == 'gzip':
buf = StringIO(response.read(len(raw)))
f = gzip.GzipFile(fileobj=buf)
body = f.read()
else:
body = response.read(len(raw))
return {
'headers': headers,
'body': body if len(body) else None
}
def process_record(self, record):
content = None
try:
payload = record.payload.read()
s = FakeSocket(payload)
response = HTTPResponse(s)
response.begin()
status_code = response.status
if status_code != 200:
return
content_type = response.getheader('Content-Type', '')
if 'text/html' not in content_type:
return
headers = response.getheaders()
content = response.read(len(payload))
except Exception:
self.increment_counter('errors', 'process_record', 1)
logging.error('Error processing record: {}', traceback.format_exc())
return
if content is not None:
content = content.strip()
if not content:
return
for item in self.process_content(record.url, headers, content):
yield item
def sendData(self, ip):
"""
套接字通信
:param ip:
:return:
"""
addr = (ip, 80)
data = "GET / HTTP/1.1\r\nHost: " + self.args.domain + "\r\n" + "Connection: close" + "\r\n\r\n"
recvdata = ""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
res = {}
res.setdefault("ip", ip)
res.setdefault("status")
res.setdefault("body")
res.setdefault("server_header")
res.setdefault("x_powered_by_header")
res.setdefault("success")
try:
sock.connect(addr)
sock.send(data)
while True:
buffer = sock.recv(1024)
if not buffer:
break
recvdata += buffer
response = HTTPResponse(FakeSocket(recvdata))
response.begin() # begin有什么用???
res["status"] = response.status
if response.status == 200: # 这里就可以保存为html文件了
msg = ip + " seems done!!!"
self.log.info(msg)
res["body"] = response.read()
res["server"] = response.getheader("Server", default="Known")
res["x_powered_by"] = response.getheader("X-Powered-By",
default="Known")
if self.args.keyword:
if self.args.keyword in res["body"]:
res["success"] = True
else:
res["success"] = False
else:
res["success"] = True
self.result.append(res)
except Exception as err:
self.log.error(err)
res["success"] = False
def http_has_valid_content_length(all_data):
"""
Gets HTTP data, returns:
1. True if it has a header "content_length" and its value <= MAX_HTTP_CONTENT_LENGTH,
2. False otherwise (includes the case were all_data doesn't contain this header)
NOTE: USE THIS ONLY ON HTTP DATA!
"""
source = FakeSocket(all_data)
response = HTTPResponse(source)
response.begin()
content_len_value = int(response.getheader('content-length', -1))
if content_len_value == -1 or content_len_value > MAX_HTTP_CONTENT_LENGTH:
print("Invalid (or missing) content-length header.")
return False
return True
def testWithoutCRLF(self):
# Tests the use of just newlines rather than CR/LFs.
data = "Echo\nthis\r\nplease"
s = ("GET / HTTP/1.0\n"
"Connection: close\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(data))
self.failUnlessEqual(response_body, data)
def testWithoutCRLF(self):
# Tests the use of just newlines rather than CR/LFs.
data = "Echo\nthis\r\nplease"
s = ("GET / HTTP/1.0\n"
"Connection: close\n"
"Content-Length: %d\n"
"\n"
"%s") % (len(data), data)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((LOCALHOST, self.port))
sock.send(s)
response = ClientHTTPResponse(sock)
response.begin()
self.failUnlessEqual(int(response.status), 200)
length = int(response.getheader('Content-Length', '0'))
response_body = response.read(length)
self.failUnlessEqual(length, len(data))
self.failUnlessEqual(response_body, data)
def isValidData(self, data):
print(MESSAGES.VALIDATING_CONTENT_LENGTH)
# Converting data to socket
s = CDataToSocket(data)
# Parsing socket to HTTP Response
response = HTTPResponse(s)
# Fetching content-length
response.begin()
content_length = response.getheader('Content-Length')
# If no conetent-length in header - blocking
if (not content_length):
return False
# DLP
language = None
try:
language = self.dataLeakPreventor.detectCode(data)
return not language
except:
return False
pass
# Reading content
content = response.read(bytearray(CHandlerHTTP.CONTENT_LENGTH_MAX))
return not (CValidatorFile.isOfficeFile(content) and \
(CHandlerHTTP.CONTENT_LENGTH_MAX < long(content_length)))
def _command(self, uri, method='GET', body='', **kwargs):
"""Makes an HTTP request through to an AirPlay server
Args:
uri(string): The URI to request
method(string): The HTTP verb to use when requesting `uri`, defaults to GET
body(string): If provided, will be sent witout alteration as the request body.
Content-Length header will be set to len(`body`)
**kwargs: If provided, Will be converted to a query string and appended to `uri`
Returns:
True: Request returned 200 OK, with no response body
False: Request returned something other than 200 OK, with no response body
Mixed: The body of the HTTP response
"""
# generate the request
if len(kwargs):
uri = uri + '?' + urlencode(kwargs)
request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str(len(body)) + "\r\n\r\n" + body
try:
request = bytes(request, 'UTF-8')
except TypeError:
pass
# send it
self.control_socket.send(request)
# parse our response
result = self.control_socket.recv(self.RECV_SIZE)
resp = HTTPResponse(FakeSocket(result))
resp.begin()
# if our content length is zero, then return bool based on result code
if int(resp.getheader('content-length', 0)) == 0:
if resp.status == 200:
return True
else:
return False
# else, parse based on provided content-type
# and return the response body
content_type = resp.getheader('content-type')
if content_type is None:
raise RuntimeError('Response returned without a content type!')
if content_type == 'text/parameters':
body = resp.read()
try:
body = str(body, 'UTF-8')
except TypeError:
pass
return email.message_from_string(body)
if content_type == 'text/x-apple-plist+xml':
return plist_loads(resp.read())
raise RuntimeError('Response received with unknown content-type: {0}'.format(content_type))
def _command(self, uri, method='GET', body='', **kwargs):
"""Makes an HTTP request through to an AirPlay server
Args:
uri(string): The URI to request
method(string): The HTTP verb to use when requesting `uri`, defaults to GET
body(string): If provided, will be sent witout alteration as the request body.
Content-Length header will be set to len(`body`)
**kwargs: If provided, Will be converted to a query string and appended to `uri`
Returns:
True: Request returned 200 OK, with no response body
False: Request returned something other than 200 OK, with no response body
Mixed: The body of the HTTP response
"""
# generate the request
if len(kwargs):
uri = uri + '?' + urlencode(kwargs)
request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str(len(body)) + "\r\n"
request+="Host: %s:%s\r\n" % (self.host,self.port)
request+="User-Agent: MediaControl/1.0\r\n"
request+="X-Apple-Session-ID: c6c0033e-96f9-11e6-b0a4-a45e60c9debb\r\n"
request+="Connection: close\r\n"
request+="\r\n"
request+=body
try:
if self.airplaySocket is None:
self.airplaySocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.airplaySocket.settimeout(self.timeout)
self.airplaySocket.connect((self.host, self.port))
except socket.error as exc:
self.airplaySocket=None
raise ValueError("Unable to connect to {0}:{1}: {2}".format(self.host, self.port, exc))
# send it
rs=self.airplaySocket.sendall(request)
# parse our response
result = self.airplaySocket.recv(self.RECV_SIZE)
resp = HTTPResponse(FakeSocket(result))
resp.begin()
# if our content length is zero, then return bool based on result code
if int(resp.getheader('content-length', 0)) == 0:
if resp.status == 200:
return True
else:
return False
# else, parse based on provided content-type
# and return the response body
content_type = resp.getheader('content-type')
if content_type is None:
raise RuntimeError('Response returned without a content type!')
return resp.read()
all_words = set([])
for fileno, f in enumerate(files):
f = warc.open(f)
record_count = 0
for record in f:
if record['Content-Type'] != 'application/http; msgtype=response':
continue
if 'WARC-Target-URI' in record:
content = record.payload.read()
src = FakeSocket(content)
response = HTTPResponse(src)
response.begin()
content_type = response.getheader('Content-Type')
if content_type is not None and "text/html" not in content_type and "xml" not in content_type and "text" not in content_type:
if "image" not in content_type:
print content_type
continue
try:
content = response.read()
h = BeautifulSoup(content, "lxml")
except IncompleteRead:
continue
for body in h.findAll("body"):
#text = [i for i in body.recursiveChildGenerator() if type(i) == NavigableString]
#text = [t.split() for t in text if len(t) > 0]
from httplib import HTTPResponse
from StringIO import StringIO
import helpers, induce
http_response_str = helpers.slurparg()
class FakeSocket():
def __init__(self, response_str):
self._file = StringIO(response_str)
def makefile(self, *args, **kwargs):
return self._file
with induce.grammar() as g:
with induce.Tracer(http_response_str, g):
source = FakeSocket(http_response_str)
response = HTTPResponse(source)
response.begin()
print "status:", response.status
print "single header:", response.getheader('Content-Type')
print "content:", response.read(len(http_response_str)) # the len here will give a 'big enough' value to read the whole content
lengths = []
num_words = []
distinct_words = []
for record in f:
if record['Content-Type'] != 'application/http; msgtype=response':
print "Skipped", record['Content-Type']
continue
if 'WARC-Target-URI' in record:
lengths.append(len(record['WARC-Target-URI']))
content = record.payload.read()
src = FakeSocket(content)
response = HTTPResponse(src)
response.begin()
content_type = response.getheader('Content-Type')
if content_type is not None and "text/html" not in content_type:
print "Skipped content", response.getheader('Content-Type')
continue
#content_lengths.append(len(content))
try:
content = response.read()
content_lengths.append(len(content))
h = BeautifulSoup(content, "lxml")
except IncompleteRead:
continue
for title in h.findAll('title'):
print "TITLE", title.text.encode('utf-8')
for body in h.findAll("body"):
def _command(self, uri, method='GET', body='', **kwargs):
"""Makes an HTTP request through to an AirPlay server
Args:
uri(string): The URI to request
method(string): The HTTP verb to use when requesting `uri`, defaults to GET
body(string): If provided, will be sent witout alteration as the request body.
Content-Length header will be set to len(`body`)
**kwargs: If provided, Will be converted to a query string and appended to `uri`
Returns:
True: Request returned 200 OK, with no response body
False: Request returned something other than 200 OK, with no response body
Mixed: The body of the HTTP response
"""
# generate the request
if len(kwargs):
uri = uri + '?' + urlencode(kwargs)
request = method + " " + uri + " HTTP/1.1\r\nContent-Length: " + str(
len(body)) + "\r\n\r\n" + body
try:
request = bytes(request, 'UTF-8')
except TypeError:
pass
# send it
self.control_socket.send(request)
# parse our response
result = self.control_socket.recv(self.RECV_SIZE)
resp = HTTPResponse(FakeSocket(result))
resp.begin()
# if our content length is zero, then return bool based on result code
if int(resp.getheader('content-length', 0)) == 0:
if resp.status == 200:
return True
else:
return False
# else, parse based on provided content-type
# and return the response body
content_type = resp.getheader('content-type')
if content_type is None:
raise RuntimeError('Response returned without a content type!')
if content_type == 'text/parameters':
body = resp.read()
try:
body = str(body, 'UTF-8')
except TypeError:
pass
return email.message_from_string(body)
if content_type == 'text/x-apple-plist+xml':
return plist_loads(resp.read())
raise RuntimeError(
'Response received with unknown content-type: {0}'.format(
content_type))
def Assertion_8_4_3(self, log):
log.AssertionID = '8.4.3'
assertion_status = log.PASS
log.assertion_log('BEGIN_ASSERTION', None)
authorization = 'on'
rq_headers = self.request_headers()
SSDP_ADDR = "239.255.255.250"
SSDP_PORT = 1900
SSDP_MX = 2
SSDP_ST = "URN:dmtf-org:service:redfish-rest:1"
#SSDP_ST = "ssdp:all"
relative_uris = self.relative_uris
##
# SSDP support is optional: check to see if the service supports/has SSDP enabled.
# The property for SSDP is defined in the 'ManagerNetworkProtocol.xml' schema
##
ServiceSupportsSSDP = False
resource = 'NetworkProtocol'
for relative_uri in relative_uris:
if resource in relative_uri:
r_url = relative_uris[relative_uri]
# GET first
json_payload, headers, status = self.http_GET(
r_url, rq_headers, authorization)
assertion_status_ = self.response_status_check(r_url, status, log)
# manage assertion status
assertion_status = log.status_fixup(assertion_status,
assertion_status_)
if assertion_status_ != log.PASS:
pass
elif not json_payload:
assertion_status = log.WARN
log.assertion_log(
'line',
'~ unable to locate URI/payload for resource: %s nested in: %s'
% (resource, r_url))
else:
# does the service support SSDP?
try:
SSDPisEnabled = json_payload['SSDP']
except:
log.assertion_log(
'line',
'~ SSDP support is optional: this service does not appear to support SSDP'
)
log.assertion_log(
'line', '~ SSDP not found in payload from GET(%s) %s' %
(r_url, rf_utility.json_string(json_payload)))
else:
try:
SSDPisEnabled = json_payload['SSDP']['ProtocolEnabled']
except:
log.assertion_log(
'line', 'GET(%s) payload: %s' %
(r_url, rf_utility.json_string(json_payload)))
log.assertion_log(
'line',
'~ ERROR: \'SSDP:ProtocolEnabled\' not found in payload'
)
assertion_status = log.FAIL
else:
if not SSDPisEnabled:
log.assertion_log(
'line', 'GET(%s) payload: %s' %
(r_url, rf_utility.json_string(json_payload)))
log.assertion_log(
'line',
'~ Note - SSDP appears to be Disabled : \'ProtocolEnabled(%s)\' is not \'true\''
% SSDPisEnabled)
assertion_status = log.WARN
else:
ServiceSupportsSSDP = True
if (ServiceSupportsSSDP == True):
ssdpRequestStr = "M-SEARCH * HTTP/1.1\r\n" + \
"MAN: \"ssdp:discover\"\r\n" + \
"HOST: %s:%d\r\n" % (SSDP_ADDR, SSDP_PORT) + \
"MX: %d\r\n" % (SSDP_MX) + \
"ST: %s\r\n" % (SSDP_ST) + "\r\n"
log.assertion_log('line', '%s' % ssdpRequestStr)
if (Python3 == True):
ssdpRequest = b'ssdpRequestStr'
else:
ssdpRequest = ssdpRequestStr
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(ssdpRequest, (SSDP_ADDR, SSDP_PORT))
except:
exc_str = sys.exc_info()[0]
assertion_status = log.WARN
log.assertion_log('line', 'Socket error %s' % exc_str)
try:
# wait for data to be returned on the M-Search for a few seconds...
few_seconds = 10
log.assertion_log(
'line',
'...wating for m-search response for %d seconds...' %
few_seconds)
ready = select.select([sock], [], [], few_seconds)
except:
exc_str = sys.exc_info()[0]
assertion_status = log.WARN
log.assertion_log('line',
'Socket/ready error %s' % exc_str)
if (assertion_status == log.PASS):
if ready[0]:
data = sock.recv(4096)
log.assertion_log('line',
'Response received: %s' % data)
# check for the 'AL' response header -- is it pointing to the Redfish Service Root URI?
hdr = HTTPResponse.getheader('AL', default=None)
if (AL_hdr == None):
log.assertion_log(
'line',
'\'AL\' header not found in m-search response')
assertion_status = log.FAIL
else:
log.assertion_log(
'line',
'...m-search response received - \'AL\' header found = %s'
% AL_hdr)
# do a get on the Redfish Service root uri...
json_payload, headers, status = self.http_GET(
AL_hdr, rq_headers, authorization)
assertion_status_ = self.response_status_check(
AL_hdr, status, log)
# manage assertion status
assertion_status = log.status_fixup(
assertion_status, assertion_status_)
else:
log.assertion_log(
'line', '...No Response from the service...')
assertion_status = log.FAIL
log.assertion_log(assertion_status, None)
return (assertion_status)
def proxy_thread(conn, client_addr):
global beganWritingToFile
# get the request from browser
request = conn.recv(MAX_DATA_RECV)
# parse the first line
first_line = request.split('\n')[0]
# get url
try:
url = first_line.split(' ')[1] # going to use the url as file name
#can't use colons in file name so have to replace with hyphen
except IndexError:
print "Caught IndexError"
return
# we don't want to cache google-analytics because:
# 1. we personally don't need analytics
# 2. we're caching files locally. Unless we had a CHRON job working in the background (which
# is beyond the scope of this project) the analytics script would be essentially useless
# 3. it greatly reduces the speed of browsing through our proxy
if (url == "http://www.google-analytics.com/analytics.js"):
print "We aint caching google analytics"
return
# if the user is requesting the search page
if (url == "http://www.search-proxy.local/"):
print "Requesting content from the search page"
file = open('./Search.html', 'r')
search_page = file.read()
conn.send("HTTP/1.1 200 OK\n")
conn.send("Content-Type: text/html\n")
conn.send("\n")
print "Sent the page header"
print search_page
conn.send(search_page)
conn.close()
return
if (url == "http://www.search-proxy.local/images/zooble.gif"):
print "Requesting content from the search page"
file = open('./zooble.gif', 'r')
zooble_image = file.read()
conn.send("HTTP/1.1 200 OK\n")
conn.send("Content-Type: image/gif\n")
conn.send("\n")
conn.send(zooble_image)
conn.close()
return
if ((url.split('/')[3]).split(':')[0] == "search"):
terms = (url.split('/')[3]).split(':')[1]
print "We received a search request"
results = search(terms, OR, CASE_INSENSITIVE, listdir(BASE_DIR))
print results
sendResults(conn, terms, results)
return
print "THE UNMODIFIED URL IS {}".format(url)
blocked_set = set(BLOCKED)
if url in blocked_set:
print "CAN'T GO THERE IT'S MALWARE!\n"
conn.send("HTTP/1.1 200 OK\n"
+"Content-Type: text/html\n"
+"\n" # Important!
+"<html><body>Sorry, the website you are attempting to access is blocked by the proxy server</body></html>\n")
conn.close()
modified_url = (url.replace(':', '')).replace('/', '')
#use the modified url as the filename when storing the file
filename = modified_url
# maximum length of a file name is 260 chars
if (len(filename) >= 260):
filename = filename[:250]
url = first_line.split(' ')[1] # going to use the url as file name
#can't use colons in file name so have to replace with hyphen
header = dict(re.findall(r"(?P<name>.*?): (?P<value>.*?)\r\n", request))
port = 80
if 'Host' in header:
webserver = header['Host']
filepath = BASE_DIR + filename
# if site was already cached
if (filename in cachedSites or os.path.exists(filepath)):
if ((calendar.timegm(time.gmtime()) - cachedSites[filename].lru) < cachedSites[filename].ttl) :
# if (1 > 0):
#if ttl hasn't expired
#increment the frequency
cachedSites[filename].frequency += 1
print "The ttl hasn't expired"
print "Reading content from file"
file = open(filepath, 'r')
content = file.read()
print "Finished reading content from file"
# then send the content to the client
conn.send(content)
return
try:
# create a socket to connect to the web server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((webserver, port))
s.send(request) # send request to webserver
while 1:
# make sure cache has enough space
if (get_size(BASE_DIR) >= MAX_CACHE_SIZE):
free_cache_memory()
return
# receive data from web server
data = s.recv(MAX_DATA_RECV)
if (len(data) > 0):
# create a temp source socket to parse the response
source = FakeSocket(data)
response = HTTPResponse(source)
response.begin()
cacheHeader = response.getheader('Cache-Control')
maxAge = 10
if (cacheHeader != 'no-cache' and (cacheHeader != None and 'must-revalidate' not in cacheHeader)):
if 'max-age' in cacheHeader:
maxAge = int(cacheHeader.split('=')[1])
# check if the file exists in the dictionary
if filename in cachedSites:
if ((calendar.timegm(time.gmtime()) - cachedSites[filename].lru) >= cachedSites[filename].ttl) :
# if (1 < 0):
#if ttl has expired
print "The ttl has expired"
freq = cachedSites[filename].frequency + 1
site = make_site(maxAge, freq, calendar.timegm(time.gmtime()))
cachedSites[filename] = site
file = open(filepath, 'a')
file.write(data)
elif beganWritingToFile == True:
#if file is being read for the first time and we're still in the process of caching it
print "Still in the process of caching the file"
file = open(filepath, 'a')
file.write(data)
# then send the data to the client
conn.send(data)
else:
print "Adding the site to the cache"
print filename
site = make_site(maxAge, 1, calendar.timegm(time.gmtime()))
cachedSites[filename] = site
beganWritingToFile = True
file = open(filepath, 'a')
file.write(data)
# then send the data to the client
conn.send(data)
else:
# just send the data to the client
print "No cache, therefore, just sending the data directly to client"
conn.send(data)
else:
beganWritingToFile = False
break
sys.exit(1)
s.close()
conn.close()
except socket.error, (value, message):
if s:
s.close()
if conn:
conn.close()
printout("Peer Reset",first_line,client_addr)
sys.exit(1)
def execute():
''' Entrypoint for HTTP call
Expect a HTTP GET request with multipart form contains two parameters: hostname and file.
hostname is a string url of server to test and file is a YAML file
Return an array of response object in JSON form
Example:
[
{
"status": [403],
"title": "913100-1"
},
{
"status": [403],
"title": "913100-2"
}
]
'''
hostname = request.values.get("hostname")
result_name = 'result.db'
if not hostname or not result_name:
return 'Must include hostname', 400
# handle uploaded YAML file
yaml_path = os.path.join(app.config['UPLOAD_PATH'], str(time.time()))
if not os.path.exists(yaml_path):
os.makedirs(yaml_path)
files = request.files.getlist('file')
for file in files:
filename = secure_filename(file.filename)
file.save(os.path.join(yaml_path, filename))
# redirect db store path to yaml_path
result_name = os.path.join(yaml_path, result_name)
# compose request
arguments = ['ftw_compatible_tool',
'-d', result_name,
'-x',
'load {} | gen | start {} | report | exit'.format(yaml_path, hostname)]
# Use threading here is necessary because parameter 'exit'
# in ftw-compatible will call sys.exit()
p = subprocess.Popen(arguments)
p.wait()
if not os.path.exists(result_name):
shutil.rmtree(yaml_path)
return 'Failed to retrieve result files', 500
# preparing json return
json_result = []
conn = sqlite3.connect(result_name)
c = conn.cursor()
c.execute('SELECT test_title, raw_response FROM Traffic')
for row in c:
test_title, raw_response = row
payload = {}
payload['title'] = test_title
if raw_response: # parse http resposne
response = HTTPResponse(FakeSocket(
raw_response.encode('ascii', 'ignore')))
response.begin()
payload["result"] = str(response.status)
if response.getheader("x-fd-int-waf-rule-hits"): # parse AFD's hitrule
payload["hitRule"] = response.getheader(
"x-fd-int-waf-rule-hits")
elif response.status == 302 and response.getheader("Location"):
res = parse_qs(urlparse(response.getheader("Location")).query)
hit_rules = reversed(
filter(None, res["modsec_ruleid"][0].split("-")))
hit_rules = filter(lambda x: int(x) > 901999, hit_rules)
hit_rules = map(lambda x: str(x), hit_rules)
hit_rules = ','.join(hit_rules)
# hit_rules = ','.join(reversed(filter(None,
# res["modsec_ruleid"][0].split("-"))))
payload["hitRule"] = hit_rules
payload["result"] = res["status"][0]
json_result.append(payload)
# clean up
conn.close()
shutil.rmtree(yaml_path)
return jsonify(json_result)
def url_open(url, headers=_DEFAULT_HEADERS, connect_timeout=5, read_timeout=4,
full_timeout=10, logger=logging.getLogger(__name__)):
""" open url
"""
resp = Response()
t = urlparse.urlparse(url)
path = t.path or '/'
if len(t.netloc.split(':')) == 2:
host, port = t.netloc.split(':')
else:
host, port = t.netloc, 80
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(connect_timeout)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
s.connect((host, port))
s.send("GET %s HTTP/1.1\r\n" % path)
s.send("Host:%s\r\n" % host)
for k,v in headers:
s.send("%s:%s\r\n" % (k, v))
s.send('\r\n')
except socket.timeout as e:
logger.exception('connection timeout: [%s]' % str(e))
resp.status_code = Response.CONNECTION_TIMEOUT
return resp
except socket.error as e:
logger.exception('socket error while send: [%s]' % str(e))
resp.status_code = Response.SOCKET_ERROR
return resp
data = []
begin = time.time()
s.settimeout(read_timeout)
while 1:
if time.time() - begin > full_timeout:
logger.exception('full timeout')
resp.status_code = Response.FULL_TIMEOUT
break
try:
msg = s.recv(4096)
if len(msg) == 0:
break
data.append(msg)
except socket.timeout as e:
logger.exception('read timeout: [%s]' % str(e))
resp.status_code = Response.READ_TIMEOUT
break
except socket.error as e:
logger.exception('socket error while read: [%s]' % str(e))
resp.status_code = Response.SOCKET_ERROR
break
try:
s.shutdown(1)
s.close()
except socket.error as e:
logger.exception('socket error while close: [%s]' % str(e))
resp.content = ''.join(data)
if resp.status_code != 200:
return resp
try:
http_resp = HTTPResponse(_FakeSocket(resp.content))
http_resp.begin()
except Exception as e:
logger.exception('read http response error: [%s]' % str(e))
resp.status_code = Response.HTTP_RESPONSE_ERROR
return resp
resp.status_code = http_resp.status
resp.content = http_resp.read(len(resp.content))
location = http_resp.getheader('location', '')
if location and str(http_resp.status)[0] == '3':
redirect_url = urlparse.urljoin(url, location)
logger.info('redirect to: [%s]' % redirect_url)
new_full_timeout = full_timeout - (time.time() - begin)
return url_open(url=redirect_url, headers=headers,
connect_timeout=connect_timeout, read_timeout=read_timeout,
full_timeout=new_full_timeout, logger=logger)
return resp
