def gen_traffic(self, url, page_source, response_headers):
if self.browser == 'chrome' or self.browser == 'chrome-headless':
request = HttpRequest(method='GET',
url=url,
headers=Traffic_generator.DEFAULT_HEADER,
body='')
if not response_headers:
# default content-type is text/html
response_headers = {'Content-Type': 'text/html'}
response = HttpResponse(code='200',
reason='OK',
headers=response_headers,
data=page_source)
return (request, response)
elif self.browser == 'phantomjs':
request = HttpRequest(method='GET',
url=url,
headers=Traffic_generator.DEFAULT_HEADER,
body='')
if not response_headers:
response_headers = {'Content-Type': 'text/html'}
response = HttpResponse(code='200',
reason='OK',
headers=response_headers,
data=page_source)
return (request, response)
def gen_traffic(self, url, page_source, response_headers):
if self.browser == 'chrome':
request = HttpRequest(method='GET',
url=url,
headers=Traffic_generator.DEFAULT_HEADER,
body='')
if response_headers is None:
response_headers = {}
response = HttpResponse(code='200',
reason='OK',
headers=response_headers,
data=page_source)
return (request, response)
# pickled error when phantomjs,the headers must be str
elif self.browser == 'phantomjs':
request = HttpRequest(method='GET',
url=url,
headers=dict2str(
Traffic_generator.DEFAULT_HEADER),
body='')
if response_headers is None:
response_headers = {}
response = HttpResponse(code='200',
reason='OK',
headers=dict2str(response_headers),
data=page_source)
return (request, response)
def gen_traffic(self, url):
domain = get_domain_from_url(url)
# add cookie to DEFAULT_HEADER
cookie = get_cookie(domain)
self.DEFAULT_HEADER['Cookie'] = cookie
# add referer
self.DEFAULT_HEADER['Referer'] = 'https"//' + domain + '/'
request = HttpRequest(method='GET', url=url, headers=self.DEFAULT_HEADER, body='')
req = urllib2.Request(url=url, headers=self.DEFAULT_HEADER)
with gevent.Timeout(10, False)as t:
try:
resp = urllib2.urlopen(req)
except urllib2.URLError, e:
REQUEST_ERROR.append(('gen_traffic()', url, e.reason))
except CertificateError:
REQUEST_ERROR.append(('gen_traffic()', url, 'ssl.CertificateError'))
def put_burp_to_trafficqueue(self):
"""
parse xxx.xml from burpsuite proxy.
:return:
"""
if os.path.exists(self.burp):
import base64
from xml.etree import cElementTree as ET
from model import HttpRequest, HttpResponse
with open(self.burp) as f:
xmlstr = f.read()
try:
root = ET.fromstring(xmlstr)
except cElementTree.ParseError, e:
print 'Parse burpsuite data error: ' + str(e)
exit(0)
for child in root:
if child.tag == 'item':
req_headers = {}
resp_headers = {}
code = ''
request, response = '', ''
for child2 in child:
if child2.tag == 'method':
method = child2.text
if child2.tag == 'url':
url = child2.text
# static url in burp
if static_reg.search(url):
break
if child2.tag == 'status':
code = child2.text
if child2.tag == 'request':
req_text = child2.text
# base64 decode
req_text = base64.b64decode(req_text)
headers_list = req_text.split(
'\r\n\r\n', 1)[0].split('\r\n')[1:]
for header in headers_list:
try:
header_key, header_value = header.split(
': ')[0], header.split(': ')[1]
if header_key not in req_headers.keys():
req_headers[header_key] = header_value
# split header error
except IndexError, e:
print e
body = req_text.split('\r\n\r\n', 1)[1]
request = HttpRequest(method, url, req_headers,
body)
if child2.tag == 'response':
resp_text = child2.text
# if response is not None
if resp_text:
# base64 decode
resp_text = base64.b64decode(resp_text)
reason = resp_text.split('\r\n')[0]
headers_list = resp_text.split(
'\r\n\r\n', 1)[0].split('\r\n')[1:]
for header in headers_list:
header_key, header_value = header.split(
': ')[0], header.split(': ')[1]
if header_key not in resp_headers.keys():
resp_headers[header_key] = header_value
data = resp_text.split('\r\n\r\n', 1)[1]
response = HttpResponse(
code, reason, resp_headers, data)
if request and response:
if request.method == 'GET' and '?' in request.url:
# filter static URL
if not static_reg.search(url):
burp_traffic.append((request, response))
traffic_queue.put((request, response))
elif request.method == 'POST' and request.body:
content_type = request.get_header('Content-Type')
# save multipart
if content_type and 'multipart/form-data; boundary=' in content_type:
MULTIPART.append((request, response))
else:
burp_traffic.append((request, response))
traffic_queue.put((request, response))