def gen_traffic(self, url, page_source, response_headers):
if self.browser == 'chrome':
request = HttpRequest(method='GET',
url=url,
headers=Traffic_generator.DEFAULT_HEADER,
body='')
if response_headers is None:
response_headers = {}
response = HttpResponse(code='200',
reason='OK',
headers=response_headers,
data=page_source)
return (request, response)
# pickled error when phantomjs,the headers must be str
elif self.browser == 'phantomjs':
request = HttpRequest(method='GET',
url=url,
headers=dict2str(
Traffic_generator.DEFAULT_HEADER),
body='')
if response_headers is None:
response_headers = {}
response = HttpResponse(code='200',
reason='OK',
headers=dict2str(response_headers),
data=page_source)
return (request, response)
def gen_traffic(self, url, page_source, response_headers):
if self.browser == 'chrome' or self.browser == 'chrome-headless':
request = HttpRequest(method='GET',
url=url,
headers=Traffic_generator.DEFAULT_HEADER,
body='')
if not response_headers:
# default content-type is text/html
response_headers = {'Content-Type': 'text/html'}
response = HttpResponse(code='200',
reason='OK',
headers=response_headers,
data=page_source)
return (request, response)
elif self.browser == 'phantomjs':
request = HttpRequest(method='GET',
url=url,
headers=Traffic_generator.DEFAULT_HEADER,
body='')
if not response_headers:
response_headers = {'Content-Type': 'text/html'}
response = HttpResponse(code='200',
reason='OK',
headers=response_headers,
data=page_source)
return (request, response)
def put_burp_to_trafficqueue(self):
"""
parse xxx.xml from burpsuite proxy.
:return:
"""
if os.path.exists(self.burp):
import base64
from xml.etree import cElementTree as ET
from model import HttpRequest, HttpResponse
with open(self.burp) as f:
xmlstr = f.read()
try:
root = ET.fromstring(xmlstr)
except cElementTree.ParseError, e:
print 'Parse burpsuite data error: ' + str(e)
exit(0)
for child in root:
if child.tag == 'item':
req_headers = {}
resp_headers = {}
code = ''
request, response = '', ''
for child2 in child:
if child2.tag == 'method':
method = child2.text
if child2.tag == 'url':
url = child2.text
# static url in burp
if static_reg.search(url):
break
if child2.tag == 'status':
code = child2.text
if child2.tag == 'request':
req_text = child2.text
# base64 decode
req_text = base64.b64decode(req_text)
headers_list = req_text.split(
'\r\n\r\n', 1)[0].split('\r\n')[1:]
for header in headers_list:
try:
header_key, header_value = header.split(
': ')[0], header.split(': ')[1]
if header_key not in req_headers.keys():
req_headers[header_key] = header_value
# split header error
except IndexError, e:
print e
body = req_text.split('\r\n\r\n', 1)[1]
request = HttpRequest(method, url, req_headers,
body)
if child2.tag == 'response':
resp_text = child2.text
# if response is not None
if resp_text:
# base64 decode
resp_text = base64.b64decode(resp_text)
reason = resp_text.split('\r\n')[0]
headers_list = resp_text.split(
'\r\n\r\n', 1)[0].split('\r\n')[1:]
for header in headers_list:
header_key, header_value = header.split(
': ')[0], header.split(': ')[1]
if header_key not in resp_headers.keys():
resp_headers[header_key] = header_value
data = resp_text.split('\r\n\r\n', 1)[1]
response = HttpResponse(
code, reason, resp_headers, data)
if request and response:
if request.method == 'GET' and '?' in request.url:
# filter static URL
if not static_reg.search(url):
burp_traffic.append((request, response))
traffic_queue.put((request, response))
elif request.method == 'POST' and request.body:
content_type = request.get_header('Content-Type')
# save multipart
if content_type and 'multipart/form-data; boundary=' in content_type:
MULTIPART.append((request, response))
else:
burp_traffic.append((request, response))
traffic_queue.put((request, response))
except BadStatusLine, e:
print e
except SocketError, e:
print e
else:
if resp.url != url:
REDIRECT.append(url)
try:
data = resp.read()
except Exception, e:
print e
else:
resp_headers = resp.headers.headers
resp_headers_dict = list2dict(resp_headers)
response = HttpResponse(code=str(resp.code),
reason=resp.msg,
headers=resp_headers_dict,
data=data)
return (request, response)
def run(self):
import gevent
from gevent import monkey
monkey.patch_all()
from gevent import pool
# default 200
# g_pool = pool.Pool(200)
g_pool = pool.Pool(self.coroutine)
tasks = [g_pool.spawn(self.gen_traffic, url) for url in self.url_list]
gevent.joinall(tasks)
traffic_list = []
for i in tasks:
