def _load_from_string(self, serialized_req_res):
try:
data = msgpack.loads(serialized_req_res, use_list=True)
except ValueError:
# ValueError: Extra data. returned when msgpack finds invalid
# data in the file
raise TraceReadException('Failed to load %s' % serialized_req_res)
try:
request_dict, response_dict, canary = data
except TypeError:
# https://github.com/andresriancho/w3af/issues/1101
# 'NoneType' object is not iterable
raise TraceReadException('Not all components found in %s' %
serialized_req_res)
if not canary == self._MSGPACK_CANARY:
# read failed, most likely because the file write is not
# complete but for some reason it was a valid msgpack file
raise TraceReadException('Invalid canary in %s' %
serialized_req_res)
request = HTTPRequest.from_dict(request_dict)
response = HTTPResponse.from_dict(response_dict)
return request, response
def load_from_file(self, _id):
fname = self._get_fname_for_id(_id)
WAIT_TIME = 0.05
#
# Due to some concurrency issues, we need to perform these checks
#
for _ in xrange(int(1 / WAIT_TIME)):
if not os.path.exists(fname):
time.sleep(WAIT_TIME)
continue
# Ok... the file exists, but it might still be being written
req_res = gzip.open(fname,
'rb',
compresslevel=self.COMPRESSION_LEVEL)
try:
data = msgpack.load(req_res, use_list=True)
except ValueError:
# ValueError: Extra data. returned when msgpack finds invalid
# data in the file
req_res.close()
time.sleep(WAIT_TIME)
continue
try:
request_dict, response_dict, canary = data
except TypeError:
# https://github.com/andresriancho/w3af/issues/1101
# 'NoneType' object is not iterable
req_res.close()
time.sleep(WAIT_TIME)
continue
if not canary == self._MSGPACK_CANARY:
# read failed, most likely because the file write is not
# complete but for some reason it was a valid msgpack file
req_res.close()
time.sleep(WAIT_TIME)
continue
# Success!
req_res.close()
request = HTTPRequest.from_dict(request_dict)
response = HTTPResponse.from_dict(response_dict)
return request, response
else:
msg = 'Timeout expecting trace file to be ready "%s"' % fname
raise DBException(msg)
def test_from_dict_encodings(self):
for body, charset in TEST_RESPONSES.values():
html = body.encode(charset)
resp = self.create_resp(Headers([("Content-Type", "text/xml")]), html)
msg = msgpack.dumps(resp.to_dict())
loaded_dict = msgpack.loads(msg)
loaded_resp = HTTPResponse.from_dict(loaded_dict)
self.assertEquals(
smart_unicode(html, DEFAULT_CHARSET, ESCAPED_CHAR, on_error_guess=False), loaded_resp.body
)
def _load_from_file(self, id):
fname = self._get_fname_for_id(id)
WAIT_TIME = 0.05
#
# Due to some concurrency issues, we need to perform these checks
#
for _ in xrange(int(1 / WAIT_TIME)):
if not os.path.exists(fname):
time.sleep(WAIT_TIME)
continue
# Ok... the file exists, but it might still be being written
req_res = open(fname, 'rb')
try:
data = msgpack.load(req_res, use_list=True)
except ValueError:
# ValueError: Extra data. returned when msgpack finds invalid
# data in the file
req_res.close()
time.sleep(WAIT_TIME)
continue
try:
request_dict, response_dict, canary = data
except TypeError:
# https://github.com/andresriancho/w3af/issues/1101
# 'NoneType' object is not iterable
req_res.close()
time.sleep(WAIT_TIME)
continue
if not canary == self._MSGPACK_CANARY:
# read failed, most likely because the file write is not
# complete but for some reason it was a valid msgpack file
req_res.close()
time.sleep(WAIT_TIME)
continue
# Success!
req_res.close()
request = HTTPRequest.from_dict(request_dict)
response = HTTPResponse.from_dict(response_dict)
return request, response
else:
msg = 'Timeout expecting trace file to be ready "%s"' % fname
raise IOError(msg)
def test_from_dict(self):
html = "header <b>ABC</b>-<b>DEF</b>-<b>XYZ</b> footer"
headers = Headers([("Content-Type", "text/html")])
orig_resp = self.create_resp(headers, html)
msg = msgpack.dumps(orig_resp.to_dict())
loaded_dict = msgpack.loads(msg)
loaded_resp = HTTPResponse.from_dict(loaded_dict)
self.assertEqual(orig_resp, loaded_resp)
orig_resp.__dict__.pop("_body_lock")
loaded_resp.__dict__.pop("_body_lock")
self.assertEqual(orig_resp.__dict__.values(), loaded_resp.__dict__.values())
def test_from_dict_encodings(self):
for body, charset in TEST_RESPONSES.values():
html = body.encode(charset)
resp = self.create_resp(Headers([('Content-Type', 'text/xml')]),
html)
msg = msgpack.dumps(resp.to_dict())
loaded_dict = msgpack.loads(msg)
loaded_resp = HTTPResponse.from_dict(loaded_dict)
self.assertEquals(
smart_unicode(html,
DEFAULT_CHARSET,
ESCAPED_CHAR,
on_error_guess=False), loaded_resp.body)
def test_from_dict(self):
html = 'header <b>ABC</b>-<b>DEF</b>-<b>XYZ</b> footer'
headers = Headers([('Content-Type', 'text/html')])
orig_resp = self.create_resp(headers, html)
msg = msgpack.dumps(orig_resp.to_dict())
loaded_dict = msgpack.loads(msg)
loaded_resp = HTTPResponse.from_dict(loaded_dict)
self.assertEqual(orig_resp, loaded_resp)
cmp_attrs = list(orig_resp.__slots__)
cmp_attrs.remove('_body_lock')
self.assertEqual({k: getattr(orig_resp, k) for k in cmp_attrs},
{k: getattr(loaded_resp, k) for k in cmp_attrs})
def test_from_dict(self):
html = 'header <b>ABC</b>-<b>DEF</b>-<b>XYZ</b> footer'
headers = Headers([('Content-Type', 'text/html')])
orig_resp = self.create_resp(headers, html)
msg = msgpack.dumps(orig_resp.to_dict())
loaded_dict = msgpack.loads(msg)
loaded_resp = HTTPResponse.from_dict(loaded_dict)
self.assertEqual(orig_resp, loaded_resp)
orig_resp.__dict__.pop('_body_lock')
loaded_resp.__dict__.pop('_body_lock')
self.assertEqual(orig_resp.__dict__.values(),
loaded_resp.__dict__.values())
def test_from_dict(self):
html = 'header <b>ABC</b>-<b>DEF</b>-<b>XYZ</b> footer'
headers = Headers([('Content-Type', 'text/html')])
orig_resp = self.create_resp(headers, html)
msg = msgpack.dumps(orig_resp.to_dict())
loaded_dict = msgpack.loads(msg)
loaded_resp = HTTPResponse.from_dict(loaded_dict)
self.assertEqual(orig_resp, loaded_resp)
cmp_attrs = list(orig_resp.__slots__)
cmp_attrs.remove('_body_lock')
self.assertEqual({k: getattr(orig_resp, k)
for k in cmp_attrs},
{k: getattr(loaded_resp, k)
for k in cmp_attrs})
def load_http_response_from_temp_file(filename, remove=True):
"""
:param filename: The filename that holds the HTTP response as msgpack
:param remove: Remove the file after reading
:return: An HTTP response instance
"""
# Importing here to prevent import cycle
from w3af.core.data.url.HTTPResponse import HTTPResponse
try:
data = msgpack.load(file(filename, 'rb'), raw=False)
result = HTTPResponse.from_dict(data)
except:
if remove:
remove_file_if_exists(filename)
raise
else:
if remove:
remove_file_if_exists(filename)
return result
def load_http_response_from_temp_file(filename, remove=True):
"""
:param filename: The filename that holds the HTTP response as msgpack
:param remove: Remove the file after reading
:return: An HTTP response instance
"""
# Importing here to prevent import cycle
from w3af.core.data.url.HTTPResponse import HTTPResponse
try:
data = msgpack.load(file(filename, 'rb'), raw=False)
result = HTTPResponse.from_dict(data)
except:
if remove:
remove_file_if_exists(filename)
raise
else:
if remove:
remove_file_if_exists(filename)
return result
def load(serialized_object):
data = msgpack.loads(serialized_object, raw=False)
return HTTPResponse.from_dict(data)
def load(serialized_object):
data = msgpack.loads(serialized_object, raw=False)
return HTTPResponse.from_dict(data)
