def gen_traffic(self, url, page_source, response_headers): if self.browser == 'chrome': request = HttpRequest(method='GET', url=url, headers=Traffic_generator.DEFAULT_HEADER, body='') if response_headers is None: response_headers = {} response = HttpResponse(code='200', reason='OK', headers=response_headers, data=page_source) return (request, response) # pickled error when phantomjs,the headers must be str elif self.browser == 'phantomjs': request = HttpRequest(method='GET', url=url, headers=dict2str( Traffic_generator.DEFAULT_HEADER), body='') if response_headers is None: response_headers = {} response = HttpResponse(code='200', reason='OK', headers=dict2str(response_headers), data=page_source) return (request, response)
def gen_traffic(self, url, page_source, response_headers): if self.browser == 'chrome' or self.browser == 'chrome-headless': request = HttpRequest(method='GET', url=url, headers=Traffic_generator.DEFAULT_HEADER, body='') if not response_headers: # default content-type is text/html response_headers = {'Content-Type': 'text/html'} response = HttpResponse(code='200', reason='OK', headers=response_headers, data=page_source) return (request, response) elif self.browser == 'phantomjs': request = HttpRequest(method='GET', url=url, headers=Traffic_generator.DEFAULT_HEADER, body='') if not response_headers: response_headers = {'Content-Type': 'text/html'} response = HttpResponse(code='200', reason='OK', headers=response_headers, data=page_source) return (request, response)
def put_burp_to_trafficqueue(self): """ parse xxx.xml from burpsuite proxy. :return: """ if os.path.exists(self.burp): import base64 from xml.etree import cElementTree as ET from model import HttpRequest, HttpResponse with open(self.burp) as f: xmlstr = f.read() try: root = ET.fromstring(xmlstr) except cElementTree.ParseError, e: print 'Parse burpsuite data error: ' + str(e) exit(0) for child in root: if child.tag == 'item': req_headers = {} resp_headers = {} code = '' request, response = '', '' for child2 in child: if child2.tag == 'method': method = child2.text if child2.tag == 'url': url = child2.text # static url in burp if static_reg.search(url): break if child2.tag == 'status': code = child2.text if child2.tag == 'request': req_text = child2.text # base64 decode req_text = base64.b64decode(req_text) headers_list = req_text.split( '\r\n\r\n', 1)[0].split('\r\n')[1:] for header in headers_list: try: header_key, header_value = header.split( ': ')[0], header.split(': ')[1] if header_key not in req_headers.keys(): req_headers[header_key] = header_value # split header error except IndexError, e: print e body = req_text.split('\r\n\r\n', 1)[1] request = HttpRequest(method, url, req_headers, body) if child2.tag == 'response': resp_text = child2.text # if response is not None if resp_text: # base64 decode resp_text = base64.b64decode(resp_text) reason = resp_text.split('\r\n')[0] headers_list = resp_text.split( '\r\n\r\n', 1)[0].split('\r\n')[1:] for header in headers_list: header_key, header_value = header.split( ': ')[0], header.split(': ')[1] if header_key not in resp_headers.keys(): resp_headers[header_key] = header_value data = resp_text.split('\r\n\r\n', 1)[1] response = HttpResponse( code, reason, resp_headers, data) if request and response: if request.method == 'GET' and '?' in request.url: # filter static URL if not static_reg.search(url): burp_traffic.append((request, response)) traffic_queue.put((request, response)) elif request.method == 'POST' and request.body: content_type = request.get_header('Content-Type') # save multipart if content_type and 'multipart/form-data; boundary=' in content_type: MULTIPART.append((request, response)) else: burp_traffic.append((request, response)) traffic_queue.put((request, response))
except BadStatusLine, e: print e except SocketError, e: print e else: if resp.url != url: REDIRECT.append(url) try: data = resp.read() except Exception, e: print e else: resp_headers = resp.headers.headers resp_headers_dict = list2dict(resp_headers) response = HttpResponse(code=str(resp.code), reason=resp.msg, headers=resp_headers_dict, data=data) return (request, response) def run(self): import gevent from gevent import monkey monkey.patch_all() from gevent import pool # default 200 # g_pool = pool.Pool(200) g_pool = pool.Pool(self.coroutine) tasks = [g_pool.spawn(self.gen_traffic, url) for url in self.url_list] gevent.joinall(tasks) traffic_list = [] for i in tasks: