def process_response(self):
"""
Parses an HTTP response after an HTTP request is sent
"""
split_response = self.response.split(self.CRLF)
response_line = split_response[0]
response_headers = {}
response_data = None
data_line = None
for line_num in range(1, len(split_response[1:])):
# CRLF represents the start of data
if split_response[line_num] == '':
data_line = line_num + 1
break
else:
# Headers are all split by ':'
header = split_response[line_num].split(':', 1)
if len(header) != 2:
raise errors.TestError(
'Did not receive a response with valid headers', {
'header_rcvd': str(header),
'function': 'http.HttpResponse.process_response'
})
response_headers[header[0].lower()] = header[1].lstrip()
if data_line is not None and data_line < len(split_response):
response_data = self.CRLF.join(split_response[data_line:])
# if the output headers say there is encoding
if 'content-encoding' in response_headers.keys():
response_data = self.parse_content_encoding(
response_headers, response_data)
if len(response_line.split(' ', 2)) != 3:
raise errors.TestError(
'The HTTP response line returned the wrong args', {
'response_line': str(response_line),
'function': 'http.HttpResponse.process_response'
})
try:
self.status = int(response_line.split(' ', 2)[1])
except ValueError:
raise errors.TestError(
'The status num of the response line isn\'t convertable', {
'response_line': str(response_line),
'function': 'http.HttpResponse.process_response'
})
self.status_msg = response_line.split(' ', 2)[2]
self.version = response_line.split(' ', 2)[0]
self.response_line = response_line
self.headers = response_headers
self.data = response_data
def __init__(self, output_dict):
self.STATUS = 'status'
self.LOG = 'log_contains'
self.NOTLOG = 'no_log_contains'
self.RESPONSE = 'response_contains'
self.ERROR = 'expect_error'
if output_dict is None:
raise errors.TestError('No output dictionary found',
{'function': 'ruleset.Output.__init__'})
self.output_dict = output_dict
skip_checks = False
if self.STATUS not in self.output_dict:
skip_checks = True
if skip_checks is False and isinstance(output_dict[self.STATUS], list):
# Check the number of integers in the list
num_ele = len(
[s for s in output_dict[self.STATUS] if type(s) is int])
# If all elements are integers, we're good
if len(output_dict[self.STATUS]) == num_ele:
self.status = output_dict[self.STATUS]
else:
raise errors.TestError(
'Non integers found in Status list', {
'status': output_dict[self.STATUS],
'function': 'ruleset.Output.__init__'
})
elif skip_checks is False and isinstance(output_dict[self.STATUS],
int):
self.status = int(output_dict[self.STATUS])
else:
self.status = None
self.response_contains_str = self.process_regex(self.RESPONSE)
self.no_log_contains_str = self.process_regex(self.NOTLOG)
self.log_contains_str = self.process_regex(self.LOG)
self.expect_error = bool(self.output_dict[self.ERROR]) if \
self.ERROR in self.output_dict and \
self.output_dict[self.ERROR] else None
if self.status is None and self.response_contains_str is None \
and self.log_contains_str is None \
and self.no_log_contains_str is None \
and self.expect_error is None:
raise errors.TestError(
'Need at least one status, response_contains ' +
', no_log_contains, or log_contains', {
'status': self.status,
'response_contains value': self.response_contains_str,
'log_contains value': self.log_contains_str,
'no_log_contains value': self.no_log_contains_str,
'expect_error value': self.expect_error,
'function': 'ruleset.Output.__init__'
})
def __init__(self, output_dict):
self.STATUS = 'status'
self.LOG = 'log_contains'
self.HTML = 'html_contains'
if output_dict is None:
raise errors.TestError('No output dictionary found',
{'function': 'ruleset.Output.__init__'})
self.output_dict = output_dict
self.status = int(output_dict[self.STATUS]) \
if self.STATUS in output_dict else None
self.html_contains_str = self.process_regex(self.HTML)
self.log_contains_str = self.process_regex(self.LOG)
if self.status is None and self.html_contains_str is None \
and self.log_contains_str is None:
raise errors.TestError(
'Need at least one status, html_contains_str ' +
'or log_contains_str', {
'status': self.status,
'html_contains_str': self.html_contains_str,
'log_contains_str': self.log_contains_str,
'function': 'ruleset.Output.__init__'
})
def __init__(self, output_dict):
self.STATUS = 'status'
self.LOG = 'log_contains'
self.NOTLOG = 'no_log_contains'
self.RESPONSE = 'response_contains'
self.ERROR = 'expect_error'
if output_dict is None:
raise errors.TestError(
'No output dictionary found',
{
'function': 'ruleset.Output.__init__'
}
)
self.output_dict = output_dict
self.status = int(output_dict[self.STATUS]) \
if self.STATUS in self.output_dict else None
self.response_contains_str = self.process_regex(self.RESPONSE)
self.no_log_contains_str = self.process_regex(self.NOTLOG)
self.log_contains_str = self.process_regex(self.LOG)
self.expect_error = bool(self.output_dict[self.ERROR]) if \
self.ERROR in self.output_dict and \
self.output_dict[self.ERROR] else None
if self.status is None and self.response_contains_str is None \
and self.log_contains_str is None \
and self.no_log_contains_str is None \
and self.expect_error is None:
raise errors.TestError(
'Need at least one status, response_contains ' +
', no_log_contains, or log_contains',
{
'status': self.status,
'response_contains value': self.response_contains_str,
'log_contains value': self.log_contains_str,
'no_log_contains value': self.no_log_contains_str,
'expect_error value': self.expect_error,
'function': 'ruleset.Output.__init__'
})
def run_stage_with_journal(self, rule_id, test, journal_file, tablename,
logger_obj):
"""
Compare entries and responses in a journal file with a logger object
This will follow similar logic as run_stage, where a logger_obj.get_logs()
MUST be implemented by the user so times can be retrieved and compared
against the responses logged in the journal db
"""
assert logger_obj is not None
conn = sqlite3.connect(journal_file)
conn.text_factory = str
cur = conn.cursor()
for i, stage in enumerate(test.stages):
'''
Query DB here for rule_id & test_title
Compare against logger_obj
'''
q = self.query_for_stage_results(tablename)
results = cur.execute(q, [i, test.test_title]).fetchall()
if len(results) == 0:
raise errors.TestError(
'SQL Query did not return results for test', {
'rule_id': rule_id,
'test': test.test_title,
'query': q,
'stage_num': i,
'function':
'testrunner.TestRunner.run_stage_with_journal'
})
result = results[0]
start = parser.parse(result[2])
end = parser.parse(result[3])
response = result[4]
status = result[5]
if (stage.output.log_contains_str
or stage.output.no_log_contains_str):
logger_obj.set_times(start, end)
lines = logger_obj.get_logs()
if stage.output.log_contains_str:
self.test_log(lines, stage.output.log_contains_str, False)
if stage.output.no_log_contains_str:
# The last argument means that we should negate the resp
self.test_log(lines, stage.output.no_log_contains_str,
True)
if stage.output.response_contains_str:
self.test_response_str(response,
stage.output.response_contains_str)
if stage.output.status:
self.test_status(stage.output.status, status)
def get_response(self):
"""
Get the response from the socket
"""
# Make socket non blocking
self.sock.setblocking(0)
our_data = []
# Beginning time
begin = time.time()
while True:
# If we have data then if we're passed the timeout break
if our_data and time.time() - begin > self.HTTP_TIMEOUT:
break
# If we're dataless wait just a bit
elif time.time() - begin > self.HTTP_TIMEOUT * 2:
break
# Recv data
try:
data = self.sock.recv(self.RECEIVE_BYTES)
if data:
our_data.append(data)
begin = time.time()
else:
# Sleep for sometime to indicate a gap
time.sleep(self.HTTP_TIMEOUT)
except socket.error as err:
# Check if we got a timeout
if err.errno == errno.EAGAIN:
pass
# SSL will return SSLWantRead instead of EAGAIN
elif (self.request_object.protocol == 'https'
and sys.exc_info()[0].__name__ == 'SSLWantReadError'):
pass
# If we didn't it's an error
else:
raise errors.TestError(
'Failed to connect to server', {
'host': self.request_object.dest_addr,
'port': self.request_object.port,
'proto': self.request_object.protocol,
'message': err,
'function': 'http.HttpUA.get_response'
})
self.response_object = HttpResponse(''.join(our_data))
try:
self.sock.shutdown(1)
self.sock.close()
except socket.error as err:
print(err)
def test_response(self, response_object, regex):
"""
Checks if the response response contains a regex specified in the
output stage. It will assert that the regex is present.
"""
if response_object is None:
raise errors.TestError(
'Searching before response received', {
'regex': regex,
'response_object': response_object,
'function': 'testrunner.TestRunner.test_response'
})
if regex.search(response_object.response):
assert True
else:
assert False
def send_request(self, http_request):
"""
Send a request and get response
"""
self.request_object = http_request
self.build_socket()
self.build_request()
try:
self.sock.send(self.request)
except socket.error as err:
raise errors.TestError(
'We were unable to send the request to the socket', {
'msg': err,
'function': 'http.HttpUA.send_request'
})
finally:
self.get_response()
def parse_content_encoding(self, response_headers, response_data):
"""
Parses a response that contains Content-Encoding to retrieve
response_data
"""
if response_headers['content-encoding'] == 'gzip':
buf = StringIO.StringIO(response_data)
zipbuf = gzip.GzipFile(fileobj=buf)
response_data = zipbuf.read()
elif response_headers['content-encoding'] == 'deflate':
data = StringIO.StringIO(zlib.decompress(response_data))
response_data = data.read()
else:
raise errors.TestError(
'Received unknown Content-Encoding', {
'content-encoding': str(
response_headers['content-encoding']),
'function': 'http.HttpResponse.parse_content_encoding'
})
return response_data
def build_socket(self):
"""
Generate either an HTTPS or HTTP socket
"""
try:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.settimeout(self.SOCKET_TIMEOUT)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Check if TLS
if self.request_object.protocol == 'https':
self.sock = ssl.wrap_socket(self.sock, ciphers=self.CIPHERS)
self.sock.connect(
(self.request_object.dest_addr, self.request_object.port))
except socket.error as msg:
raise errors.TestError(
'Failed to connect to server', {
'host': self.request_object.dest_addr,
'port': self.request_object.port,
'proto': self.request_object.protocol,
'message': msg,
'function': 'http.HttpUA.build_socket'
})
def build_request(self):
request = '#method# #uri##version#%s#headers#%s#data#' % \
(self.CRLF, self.CRLF)
request = string.replace(request, '#method#',
self.request_object.method)
# We add a space after here to account for HEAD requests with no url
request = string.replace(request, '#uri#',
self.request_object.uri + ' ')
request = string.replace(request, '#version#',
self.request_object.version)
# Expand out our headers into a string
headers = ''
if self.request_object.headers != {}:
for hname, hvalue in self.request_object.headers.iteritems():
headers += str(hname) + ': ' + str(hvalue) + str(self.CRLF)
request = string.replace(request, '#headers#', headers)
# If we have data append it
if self.request_object.data != '':
data = str(self.request_object.data)
request = string.replace(request, '#data#', data)
else:
request = string.replace(request, '#data#', '')
# If we have a Raw Request we should use that instead
if self.request_object.raw_request is not None:
if self.request_object.encoded_request is not None:
raise errors.TestError(
'Cannot specify both raw and encoded modes',
{'function': 'http.HttpUA.build_request'})
request = self.request_object.raw_request
# Check for newlines (without CR prior)
request = re.sub(r'(?<!x)\n', self.CRLF, request)
request = request.decode('string_escape')
if self.request_object.encoded_request is not None:
request = base64.b64decode(self.request_object.encoded_request)
request = request.decode('string_escape')
# if we have an Encoded request we should use that
self.request = request
def check_for_cookie(self, cookie):
# http://bayou.io/draft/cookie.domain.html
# Check if our originDomain is an IP
origin_is_ip = True
try:
IP(self.dest_addr)
except ValueError:
origin_is_ip = False
for cookie_morsals in cookie.values():
# If the coverdomain is blank or the domain is an IP set the domain to be the origin
if cookie_morsals['domain'] == '' or origin_is_ip:
# We want to always add a domain so it's easy to parse later
return (cookie, self.dest_addr)
# If the coverdomain is set it can be any subdomain
else:
cover_domain = cookie_morsals['domain']
# strip leading dots
# Find all leading dots not just first one
# http://tools.ietf.org/html/rfc6265#section-4.1.2.3
first_non_dot = 0
for i in range(len(cover_domain)):
if cover_domain[i] != '.':
first_non_dot = i
break
cover_domain = cover_domain[first_non_dot:]
# We must parse the coverDomain to make sure its not in the suffix list
psl_path = os.path.dirname(__file__) + os.path.sep + \
'util' + os.path.sep + 'public_suffix_list.dat'
# Check if the public suffix list is present in the ftw dir
if os.path.exists(psl_path):
pass
else:
raise errors.TestError(
'unable to find the needed public suffix list', {
'Search_Dir': os.path.dirname(__file__),
'function': 'http.HttpResponse.check_for_cookie'
})
try:
with open(psl_path, 'r') as public_suffixs:
for line in public_suffixs:
if line[:2] == '//' or line[0] == ' ' or line[
0].strip() == '':
continue
if cover_domain == line.strip():
return False
except IOError:
raise errors.TestError(
'unable to open the needed publix suffix list', {
'path': psl_path,
'function': 'http.HttpResponse.check_for_cookie'
})
# Generate Origin Domain TLD
i = self.dest_addr.rfind('.')
o_tld = self.dest_addr[i + 1:]
# if our cover domain is the origin TLD we ignore
# Quick sanity check
if cover_domain == o_tld:
return False
# check if our coverdomain is a subset of our origin domain
# Domain match (case insensative)
if cover_domain == self.dest_addr:
return (cookie, self.dest_addr)
# Domain match algorithm (rfc2965)
bvalue = cover_domain.lower()
hdn = self.dest_addr.lower()
nend = hdn.find(bvalue)
if nend is not False:
nvalue = hdn[0:nend]
# Modern browsers don't care about dot
if nvalue[-1] == '.':
nvalue = nvalue[0:-1]
else:
# We don't have an address of the form
return False
if nvalue == '':
return False
return (cookie, self.dest_addr)
def get_response(self):
"""
Get the response from the socket
"""
self.sock.setblocking(0)
our_data = []
# Beginning time
begin = time.time()
while True:
# If we have data then if we're passed the timeout break
if our_data and time.time() - begin > self.HTTP_TIMEOUT:
break
# If we're dataless wait just a bit
elif time.time() - begin > self.HTTP_TIMEOUT * 2:
break
# Recv data
try:
data = self.sock.recv(self.RECEIVE_BYTES)
if data:
our_data.append(data)
begin = time.time()
else:
# Sleep for sometime to indicate a gap
time.sleep(self.HTTP_TIMEOUT)
except socket.error as err:
# Check if we got a timeout
if err.errno == errno.EAGAIN:
pass
# SSL will return SSLWantRead instead of EAGAIN
elif sys.platform == 'win32' and \
err.errno == errno.WSAEWOULDBLOCK:
pass
elif (self.request_object.protocol == 'https'
and err[0] == ssl.SSL_ERROR_WANT_READ):
continue
# If we didn't it's an error
else:
raise errors.TestError(
'Failed to connect to server', {
'host': self.request_object.dest_addr,
'port': self.request_object.port,
'proto': self.request_object.protocol,
'message': err,
'function': 'http.HttpUA.get_response'
})
if ''.join(our_data) == '':
raise errors.TestError(
'No response from server. Request likely timed out.', {
'host': self.request_object.dest_addr,
'port': self.request_object.port,
'proto': self.request_object.protocol,
'msg': 'Please send the request and check Wireshark',
'function': 'http.HttpUA.get_response'
})
self.response_object = HttpResponse(''.join(our_data), self)
try:
self.sock.shutdown(1)
self.sock.close()
except socket.error as err:
raise errors.TestError(
'We were unable to close the socket as expected.', {
'msg': err,
'function': 'http.HttpUA.get_response'
})
def build_request(self):
request = '#method# #uri##version#%s#headers#%s#data#' % \
(self.CRLF, self.CRLF)
request = string.replace(request, '#method#',
self.request_object.method)
# We add a space after here to account for HEAD requests with no url
request = string.replace(request, '#uri#',
self.request_object.uri + ' ')
request = string.replace(request, '#version#',
self.request_object.version)
available_cookies = self.find_cookie()
# If the user has requested a tracked cookie and we have one set it
if available_cookies:
cookie_value = ''
if 'cookie' in self.request_object.headers.keys():
# Create a SimpleCookie out of our provided cookie
try:
provided_cookie = Cookie.SimpleCookie()
provided_cookie.load(self.request_object.headers['cookie'])
except Cookie.CookieError as err:
raise errors.TestError(
'Error processing the existing cookie into a SimpleCookie',
{
'msg':
str(err),
'set_cookie':
str(self.request_object.headers['cookie']),
'function':
'http.HttpResponse.build_request'
})
result_cookie = {}
for cookie_key, cookie_morsal in provided_cookie.iteritems():
result_cookie[cookie_key] = provided_cookie[
cookie_key].value
for cookie in available_cookies:
for cookie_key, cookie_morsal in cookie.iteritems():
if cookie_key in result_cookie.keys():
# we don't overwrite a user specified cookie with a saved one
pass
else:
result_cookie[cookie_key] = cookie[
cookie_key].value
for key, value in result_cookie.iteritems():
cookie_value += (unicode(key) + '=' + unicode(value) +
'; ')
# Remove the trailing semicolon
cookie_value = cookie_value[:-2]
self.request_object.headers['cookie'] = cookie_value
else:
for cookie in available_cookies:
for cookie_key, cookie_morsal in cookie.iteritems():
cookie_value += (unicode(cookie_key) + '=' +
unicode(cookie_morsal.coded_value) +
'; ')
# Remove the trailing semicolon
cookie_value = cookie_value[:-2]
self.request_object.headers['cookie'] = cookie_value
# Expand out our headers into a string
headers = ''
if self.request_object.headers != {}:
for hname, hvalue in self.request_object.headers.iteritems():
hname, hvalue = str(hname), str(hvalue)
headers += unicode(hname) + ': ' + unicode(hvalue) + self.CRLF
request = string.replace(request, '#headers#', headers)
# If we have data append it
if self.request_object.data != '':
# Before we do that see if that is a charset
encoding = "utf-8"
# Check to see if we have a content type and magic is off (otherwise UTF-8)
if 'Content-Type' in self.request_object.headers.keys(
) and self.request_object.stop_magic is False:
pattern = re.compile(r'\;\s{0,1}?charset\=(.*?)(?:$|\;|\s)')
m = re.search(pattern,
self.request_object.headers['Content-Type'])
if m:
possible_choices = list(
set(encodings.aliases.aliases.keys())) + list(
set(encodings.aliases.aliases.values()))
choice = m.group(1)
# Python will allow these aliases but doesn't list them
choice = choice.replace('-', '_')
choice = choice.lower()
if choice in possible_choices:
encoding = choice
try:
data = self.request_object.data.encode(encoding)
except UnicodeEncodeError as err:
raise errors.TestError(
'Error encoding the data with the charset specified', {
'msg':
str(err),
'Content-Type':
str(self.request_object.headers['Content-Type']),
'data':
unicode(self.request_object.data),
'function':
'http.HttpResponse.build_request'
})
request = string.replace(request, '#data#', data)
else:
request = string.replace(request, '#data#', '')
# If we have a Raw Request we should use that instead
if self.request_object.raw_request is not None:
if self.request_object.encoded_request is not None:
raise errors.TestError(
'Cannot specify both raw and encoded modes',
{'function': 'http.HttpUA.build_request'})
request = self.request_object.raw_request
# We do this regardless of magic if you want to send a literal
# '\' 'r' or 'n' use encoded request.
request = request.decode('string_escape')
if self.request_object.encoded_request is not None:
request = base64.b64decode(self.request_object.encoded_request)
request = request.decode('string_escape')
# if we have an Encoded request we should use that
self.request = request
def process_response(self):
"""
Parses an HTTP response after an HTTP request is sent
"""
split_response = self.response.split(self.CRLF)
response_line = split_response[0]
response_headers = {}
response_data = None
data_line = None
for line_num in range(1, len(split_response[1:])):
# CRLF represents the start of data
if split_response[line_num] == '':
data_line = line_num + 1
break
else:
# Headers are all split by ':'
header = split_response[line_num].split(':', 1)
if len(header) != 2:
raise errors.TestError(
'Did not receive a response with valid headers', {
'header_rcvd': str(header),
'function': 'http.HttpResponse.process_response'
})
response_headers[header[0].lower()] = header[1].lstrip()
if 'set-cookie' in response_headers.keys():
try:
cookie = Cookie.SimpleCookie()
cookie.load(response_headers['set-cookie'])
except Cookie.CookieError as err:
raise errors.TestError(
'Error processing the cookie content into a SimpleCookie',
{
'msg': str(err),
'set_cookie': str(response_headers['set-cookie']),
'function': 'http.HttpResponse.process_response'
})
# if the check_for_cookie is invalid then we don't save it
if self.check_for_cookie(cookie) is False:
raise errors.TestError(
'An invalid cookie was specified', {
'set_cookie': str(response_headers['set-cookie']),
'function': 'http.HttpResponse.process_response'
})
else:
self.cookiejar.append((cookie, self.dest_addr))
if data_line is not None and data_line < len(split_response):
response_data = self.CRLF.join(split_response[data_line:])
# if the output headers say there is encoding
if 'content-encoding' in response_headers.keys():
response_data = self.parse_content_encoding(
response_headers, response_data)
if len(response_line.split(' ', 2)) != 3:
raise errors.TestError(
'The HTTP response line returned the wrong args', {
'response_line': str(response_line),
'function': 'http.HttpResponse.process_response'
})
try:
self.status = int(response_line.split(' ', 2)[1])
except ValueError:
raise errors.TestError(
'The status num of the response line isn\'t convertable', {
'msg': 'This may be an HTTP 1.0 \'Simple Req\\Res\', it \
doesn\'t have HTTP headers and FTW will not parse these',
'response_line': str(response_line),
'function': 'http.HttpResponse.process_response'
})
self.status_msg = response_line.split(' ', 2)[2]
self.version = response_line.split(' ', 2)[0]
self.response_line = response_line
self.headers = response_headers
self.data = response_data
