def handle_sequenceRunnerRunButton_clicked(self):
""" Run a sequence """
if 'Cancel' == self.mainWindow.sequenceRunnerRunButton.text(
) and self.pending_sequence_requests is not None:
self.cancel_sequence_requests = True
for context, pending_request in self.pending_sequence_requests.items(
):
pending_request.cancel()
self.pending_sequence_requests = None
self.mainWindow.sequenceRunnerButton.setText('Send')
self.mainWindow.sequenceRunnerButton.setValue(0)
return
self.sequenceRunnerDataModel.clearModel()
sequenceId = str(
self.mainWindow.sequenceRunnerSequenceComboBox.itemData(
self.mainWindow.sequenceRunnerSequenceComboBox.currentIndex()))
use_global_cookie_jar = self.mainWindow.sequenceRunnerUseGlobalCookieJar.isChecked(
)
if use_global_cookie_jar:
self.sequenceRunnerCookieJar = self.framework.get_global_cookie_jar(
)
else:
self.sequenceRunnerCookieJar = InMemoryCookieJar(
self.framework, self)
self.sequence_requestRunner = RequestRunner(self.framework, self)
self.sequence_requestRunner.setup(
self.sequence_runner_response_received,
self.sequenceRunnerCookieJar, sequenceId)
self.pending_sequence_requests = self.sequence_requestRunner.run_sequence(
)
self.mainWindow.sequenceRunnerRunButton.setText('Cancel')
def do_startSpidering(self):
print('do_startSpidering')
# TODO: decide about global cookies ?
self.requestRunner = RequestRunner(self.framework, self)
self.requestRunner.setup(self.network_response_received,
self.cookieJar, self.sequence_id)
self.keep_spidering = True
self.renderer_available = False
self.handle_spider_available()
def requester_send_button_clicked(self):
""" Make a request from the Request tab """
if 'Cancel' == self.mainWindow.requesterSendButton.text() and self.pending_request is not None:
self.pending_request.cancel()
self.pending_request = None
self.mainWindow.requesterSendButton.setText('Send')
return
qurl = QUrl.fromUserInput(self.mainWindow.requesterUrlEdit.text())
url = qurl.toEncoded().data().decode('utf-8')
self.mainWindow.requesterUrlEdit.setText(url)
self.framework.set_raft_config_value('requesterUrlEdit', url)
templateText = str(self.mainWindow.requesterTemplateEdit.toPlainText())
method = str(self.mainWindow.requesterRequestMethod.currentText())
use_global_cookie_jar = self.mainWindow.requesterUseGlobalCookieJar.isChecked()
replacements = self.build_replacements(method, url)
(method, url, headers, body) = self.process_template(url, templateText, replacements)
sequenceId = None
if self.mainWindow.requesterSequenceCheckBox.isChecked():
sequenceId = str(self.mainWindow.requesterSequenceComboBox.itemData(self.mainWindow.requesterSequenceComboBox.currentIndex()))
self.requestRunner = RequestRunner(self.framework, self)
if use_global_cookie_jar:
self.requesterCookieJar = self.framework.get_global_cookie_jar()
else:
self.requesterCookieJar = InMemoryCookieJar(self.framework, self)
self.requestRunner.setup(self.requester_response_received, self.requesterCookieJar, sequenceId)
self.pending_request = self.requestRunner.queue_request(method, url, headers, body)
self.mainWindow.requesterSendButton.setText('Cancel')
self.miniResponseRenderWidget.clear_response_render()
def do_startSpidering(self):
print('do_startSpidering')
# TODO: decide about global cookies ?
self.requestRunner = RequestRunner(self.framework, self)
self.requestRunner.setup(self.network_response_received, self.cookieJar, self.sequence_id)
self.keep_spidering = True
self.renderer_available = False
self.handle_spider_available()
def requester_send_button_clicked(self):
""" Make a request from the Request tab """
if 'Cancel' == self.mainWindow.requesterSendButton.text(
) and self.pending_request is not None:
self.pending_request.cancel()
self.pending_request = None
self.mainWindow.requesterSendButton.setText('Send')
return
qurl = QUrl.fromUserInput(self.mainWindow.requesterUrlEdit.text())
url = qurl.toEncoded().data().decode('utf-8')
self.mainWindow.requesterUrlEdit.setText(url)
self.framework.set_raft_config_value('requesterUrlEdit', url)
templateText = str(self.mainWindow.requesterTemplateEdit.toPlainText())
method = str(self.mainWindow.requesterRequestMethod.currentText())
use_global_cookie_jar = self.mainWindow.requesterUseGlobalCookieJar.isChecked(
)
replacements = self.build_replacements(method, url)
(method, url, headers,
body) = self.process_template(url, templateText, replacements)
sequenceId = None
if self.mainWindow.requesterSequenceCheckBox.isChecked():
sequenceId = str(
self.mainWindow.requesterSequenceComboBox.itemData(
self.mainWindow.requesterSequenceComboBox.currentIndex()))
self.requestRunner = RequestRunner(self.framework, self)
if use_global_cookie_jar:
self.requesterCookieJar = self.framework.get_global_cookie_jar()
else:
self.requesterCookieJar = InMemoryCookieJar(self.framework, self)
self.requestRunner.setup(self.requester_response_received,
self.requesterCookieJar, sequenceId)
self.pending_request = self.requestRunner.queue_request(
method, url, headers, body)
self.mainWindow.requesterSendButton.setText('Cancel')
self.miniResponseRenderWidget.clear_response_render()
def handle_sequenceRunnerRunButton_clicked(self):
""" Run a sequence """
if 'Cancel' == self.mainWindow.sequenceRunnerRunButton.text() and self.pending_sequence_requests is not None:
self.cancel_sequence_requests = True
for context, pending_request in self.pending_sequence_requests.items():
pending_request.cancel()
self.pending_sequence_requests = None
self.mainWindow.sequenceRunnerButton.setText('Send')
self.mainWindow.sequenceRunnerButton.setValue(0)
return
self.sequenceRunnerDataModel.clearModel()
sequenceId = str(self.mainWindow.sequenceRunnerSequenceComboBox.itemData(self.mainWindow.sequenceRunnerSequenceComboBox.currentIndex()))
use_global_cookie_jar = self.mainWindow.sequenceRunnerUseGlobalCookieJar.isChecked()
if use_global_cookie_jar:
self.sequenceRunnerCookieJar = self.framework.get_global_cookie_jar()
else:
self.sequenceRunnerCookieJar = InMemoryCookieJar(self.framework, self)
self.sequence_requestRunner = RequestRunner(self.framework, self)
self.sequence_requestRunner.setup(self.sequence_runner_response_received, self.sequenceRunnerCookieJar, sequenceId)
self.pending_sequence_requests = self.sequence_requestRunner.run_sequence()
self.mainWindow.sequenceRunnerRunButton.setText('Cancel')
class SpiderThread(QThread):
def __init__(self, framework, queueDataModel, pendingResponsesDataModel, pendingAnalysisDataModel, internalStateDataModel, parent = None):
QThread.__init__(self, parent)
self.framework = framework
self.queueDataModel = queueDataModel
self.pendingResponsesDataModel = pendingResponsesDataModel
self.pendingAnalysisDataModel = pendingAnalysisDataModel
self.internalStateDataModel = internalStateDataModel
self.qlock = QMutex()
self.qlock_analysis = QMutex()
QObject.connect(self, SIGNAL('quit()'), self.quitHandler)
QObject.connect(self, SIGNAL('started()'), self.startedHandler)
def do_setup(self):
self.spider_items = {}
self.spider_outstanding_requests = {}
self.analysis_queue = deque()
self.scopeController = self.framework.getScopeController()
self.contentExtractor = self.framework.getContentExtractor()
self.htmlExtractor = self.contentExtractor.getExtractor('html')
self.spiderConfig = self.framework.getSpiderConfig()
self.spiderRules = SpiderRules(self.framework, self)
self.formFiller = FormFiller(self.framework, self)
self.re_location_header = re.compile(r'^Location:\s*(.+)$', re.I)
self.re_content_location_header = re.compile(r'^Content-Location:\s*(.+)$', re.I)
self.Data = None
self.read_cursor = None
self.read_cursor2 = None
self.write_cursor = None
self.keep_spidering = False
def db_attach(self):
self.Data = self.framework.getDB()
self.read_cursor = self.Data.allocate_thread_cursor()
self.read_cursor2 = self.Data.allocate_thread_cursor()
self.write_cursor = self.Data.allocate_thread_cursor()
self.populateExistingSpiderData()
def db_detach(self):
self.close_cursor()
self.Data = None
def close_cursor(self):
if self.write_cursor and self.Data:
self.write_cursor.close()
self.Data.release_thread_cursor(self.write_cursor)
self.write_cursor = None
if self.read_cursor2 and self.Data:
self.read_cursor2.close()
self.Data.release_thread_cursor(self.read_cursor2)
self.read_cursor2 = None
if self.read_cursor and self.Data:
self.read_cursor.close()
self.Data.release_thread_cursor(self.read_cursor)
self.read_cursor = None
def run(self):
QObject.connect(self, SIGNAL('populateExistingSpiderData()'), self.do_populateExistingSpiderData, Qt.DirectConnection)
QObject.connect(self, SIGNAL('clearSpiderQueue()'), self.do_clearSpiderQueue, Qt.DirectConnection)
QObject.connect(self, SIGNAL('clearSpiderPendingResponses()'), self.do_clearSpiderPendingResponses, Qt.DirectConnection)
QObject.connect(self, SIGNAL('resetSpiderPendingResponses()'), self.do_resetSpiderPendingResponses, Qt.DirectConnection)
QObject.connect(self, SIGNAL('startSpidering()'), self.do_startSpidering, Qt.DirectConnection)
QObject.connect(self, SIGNAL('stopSpidering()'), self.do_stopSpidering, Qt.DirectConnection)
QObject.connect(self, SIGNAL('spiderItemFinished()'), self.do_spiderItemFinished, Qt.DirectConnection)
QObject.connect(self, SIGNAL('generateSpiderValues()'), self.do_generateSpiderValues, Qt.DirectConnection)
QObject.connect(self, SIGNAL('sendNextSpiderRequest()'), self.do_sendNextSpiderRequest, Qt.DirectConnection)
QObject.connect(self, SIGNAL('addPendingAnalysis()'), self.do_addPendingAnalysis, Qt.DirectConnection)
self.exec_()
def quitHandler(self):
self.framework.debug_log('SpiderThread quit...')
self.close_cursor()
self.exit(0)
def startedHandler(self):
self.framework.debug_log('SpiderThread started...')
self.do_setup()
self.framework.subscribe_database_events(self.db_attach, self.db_detach)
self.framework.subscribe_populate_spider_response_id(self.do_populate_spider_response_id)
self.framework.subscribe_populate_spider_response_list(self.do_populate_spider_response_list)
def populateExistingSpiderData(self):
QTimer.singleShot(50, self, SIGNAL('populateExistingSpiderData()'))
def clearSpiderQueue(self):
QTimer.singleShot(50, self, SIGNAL('clearSpiderQueue()'))
def clearSpiderPendingResponses(self):
QTimer.singleShot(50, self, SIGNAL('clearSpiderPendingResponses()'))
def resetSpiderPendingResponses(self):
QTimer.singleShot(50, self, SIGNAL('resetSpiderPendingResponses()'))
def startSpidering(self, spider_callback, sequence_id, cookieJar):
print('startSpidering')
self.spider_callback = spider_callback
if sequence_id and sequence_id > 0:
self.sequence_id = sequence_id
else:
self.sequence_id = None
self.cookieJar = cookieJar
QTimer.singleShot(100, self, SIGNAL('startSpidering()'))
def stopSpidering(self):
print('stopSpidering')
self.keep_spidering = False
QTimer.singleShot(50, self, SIGNAL('stopSpidering()'))
def spiderItemFinished(self, response_id):
###-> print('*****1')
self.qlock.lock()
###-> print('*****2')
try:
###-> print('*****3')
self.Data.update_spider_pending_response_id(self.write_cursor, 'C', response_id, 'render')
###-> print('*****4')
finally:
self.qlock.unlock()
###-> print('*****5')
self.handle_spider_available()
def do_startSpidering(self):
print('do_startSpidering')
# TODO: decide about global cookies ?
self.requestRunner = RequestRunner(self.framework, self)
self.requestRunner.setup(self.network_response_received, self.cookieJar, self.sequence_id)
self.keep_spidering = True
self.renderer_available = False
self.handle_spider_available()
def do_spiderItemFinished(self):
pass
###-> print('****6')
def handle_spider_available(self):
if self.keep_spidering:
QTimer.singleShot(10, self, SIGNAL('generateSpiderValues()'))
do_send = False
self.qlock.lock()
try:
if len(self.spider_outstanding_requests) == 0:
do_send = True
finally:
self.qlock.unlock()
if do_send:
QTimer.singleShot(10, self, SIGNAL('sendNextSpiderRequest()'))
self.dispatch_next_render_item()
def do_populate_spider_response_id(self, response_id):
self.qlock.lock()
try:
self.add_pending_spider_response(response_id, 0)
finally:
self.qlock.unlock()
QTimer.singleShot(50, self, SIGNAL('generateSpiderValues()'))
def do_populate_spider_response_list(self, id_list):
self.qlock.lock()
try:
for response_id in id_list:
self.add_pending_spider_response(int(response_id), 0)
self.add_pending_spider_response(int(response_id), 0)
finally:
self.qlock.unlock()
QTimer.singleShot(50, self, SIGNAL('generateSpiderValues()'))
def add_pending_spider_response(self, response_id, depth):
row = self.Data.read_responses_by_id(self.read_cursor, response_id)
if not row:
self.framework.log_warning('missing response id: %s' % (response_id))
return
response_items = [m or '' for m in row]
content_type = str(response_items[ResponsesTable.RES_CONTENT_TYPE])
content_type, charset = self.contentExtractor.parseContentType(content_type)
base_type = self.contentExtractor.getBaseType(content_type)
if 'html' == base_type:
self.add_pending_spider_response_id(response_id, 'spider', depth)
self.add_pending_spider_response_id(response_id, 'render', depth)
else:
# TODO: implement other render types
self.add_pending_spider_response_id(response_id, 'spider', depth)
self.framework.log_warning('skipping unsupported type for render analysis for [%s]: %s' % (response_id, content_type))
def add_pending_spider_response_id(self, response_id, request_type, depth):
data_item = [response_id, request_type, depth, 'P']
if self.Data.add_spider_pending_response_id(self.write_cursor, data_item):
self.pendingResponsesDataModel.append_data([data_item])
def do_generateSpiderValues(self):
self.generate_from_pending_responses()
self.generate_from_pending_analysis()
def generate_from_pending_responses(self):
self.qlock.lock()
keep_looping = True
try:
putback_rows = []
print('generate spider values from pending responses')
while keep_looping:
data_item = self.pendingResponsesDataModel.popleft_data()
###-> print(data_item)
if data_item:
response_id, request_type, depth, status = data_item
else:
keep_looping = False
if data_item is not None:
if 'spider' == request_type:
self.generate_spider_values(response_id, depth)
# remove from database
self.Data.update_spider_pending_response_id(self.write_cursor, 'C', response_id, request_type)
else:
# put back
putback_rows.append(data_item)
self.pendingResponsesDataModel.append_data(putback_rows)
except Exception as error:
self.framework.report_exception(error)
finally:
self.qlock.unlock()
def generate_from_pending_analysis(self):
keep_looping = True
self.qlock.lock()
try:
print('generate spider values from analysis queues')
while self.keep_spidering and keep_looping:
data_item = self.pendingAnalysisDataModel.popleft_data()
###-> print(data_item)
if data_item:
analysis_id, analysis_type, content, url, depth = data_item
if depth:
new_depth = int(depth) + 1
else:
new_depth = 1
if new_depth < self.spiderConfig.max_link_depth:
spider_requests = self.calculate_spider_requests_from_analysis(data_item)
self.add_spider_requests(spider_requests, url, new_depth)
self.Data.delete_spider_pending_analysis(self.write_cursor, analysis_id)
else:
keep_looping = False
except Exception as error:
self.framework.report_exception(error)
finally:
self.qlock.unlock()
def dispatch_next_render_item(self):
###-> print('****7')
if self.keep_spidering:
###-> print('****8')
render_item = self.get_next_render_item()
if render_item:
self.renderer_available = False
QObject.emit(self.spider_callback, SIGNAL('spiderItemAvailable(int, QString, QUrl, int)'), render_item[0], render_item[1], render_item[2], render_item[3])
else:
self.renderer_available = True
def do_sendNextSpiderRequest(self):
while True:
spider_request = self.get_next_spider_request()
###-> print(spider_request)
if not spider_request:
return
if spider_request:
method, url, headers, body, context = spider_request
if self.scopeController.isUrlInScope(url, url):
self.requestRunner.queue_request(method, url, headers, body, context)
return
else:
self.framework.log_warning('SKIPPING out of scope: [%s]' % (url))
self.qlock.lock()
try:
data_item = self.spider_outstanding_requests.pop(context)
self.Data.update_spider_queue_item_status(self.write_cursor, int(data_item[SpiderQueueTable.ID]), 'C')
except KeyError:
pass
finally:
self.qlock.unlock()
def network_response_received(self, response_id, context):
data_item = None
context = str(context)
if context:
self.qlock.lock()
try:
if context not in self.spider_outstanding_requests:
self.framework.log_warning('*** missing spider request for [%s]' % (context))
else:
data_item = self.spider_outstanding_requests.pop(context)
self.Data.update_spider_queue_item_status(self.write_cursor, int(data_item[SpiderQueueTable.ID]), 'C')
self.add_pending_spider_response_id(response_id, 'spider', int(data_item[SpiderQueueTable.DEPTH]))
self.add_pending_spider_response_id(response_id, 'render', int(data_item[SpiderQueueTable.DEPTH]))
finally:
self.qlock.unlock()
if self.keep_spidering:
QTimer.singleShot(50, self, SIGNAL('generateSpiderValues()'))
QTimer.singleShot(50, self, SIGNAL('sendNextSpiderRequest()'))
# TODO: checking concurrency issues
if self.renderer_available:
self.dispatch_next_render_item()
def get_next_render_item(self):
render_item = None
self.qlock.lock()
keep_looping = True
try:
putback_rows = []
while keep_looping:
data_item = self.pendingResponsesDataModel.popleft_data()
###-> print(data_item)
if data_item:
response_id, request_type, depth, status = data_item
else:
keep_looping = False
if data_item is not None:
if 'render' == request_type:
row = self.Data.read_responses_by_id(self.read_cursor, response_id)
if not row:
self.framework.log_warning('missing response id: %s' % (response_id))
continue
response_items = [m or '' for m in row]
qurl = QUrl.fromEncoded(response_items[ResponsesTable.URL])
dataContent = str(response_items[ResponsesTable.RES_DATA])
render_item = (response_id, dataContent, qurl, depth)
keep_looping = False
else:
# put back
putback_rows.append(data_item)
self.pendingResponsesDataModel.appendleft_data(putback_rows)
except Exception as error:
self.framework.report_exception(error)
finally:
self.qlock.unlock()
###--> print('next render item', render_item)
return render_item
def get_next_spider_request(self):
self.qlock.lock()
spider_request = None
try:
data_item = self.queueDataModel.popleft_data()
if data_item:
method, target_url, headers, body = self.make_spider_request_content(data_item)
context = uuid.uuid4().hex
self.spider_outstanding_requests[context] = data_item
spider_request = (method, target_url, headers, body, context)
finally:
self.qlock.unlock()
###--> print('next spider_request item', spider_request)
return spider_request
def do_stopSpidering(self):
self.keep_spidering = False
print(('do_stopSpidering', self, self.keep_spidering))
def generate_spider_values(self, response_id, depth):
new_depth = depth + 1
if new_depth >= self.spiderConfig.max_link_depth:
return
row = self.Data.read_responses_by_id(self.read_cursor, response_id)
if not row:
self.framework.log_warning('missing response id: %s' % (response_id))
return
response_items = [m or '' for m in row]
url = str(response_items[ResponsesTable.URL])
response_headers = str(response_items[ResponsesTable.RES_HEADERS])
response_body = str(response_items[ResponsesTable.RES_DATA])
content_type = str(response_items[ResponsesTable.RES_CONTENT_TYPE])
spider_requests = self.calculate_spider_requests(url, response_headers, response_body, content_type, new_depth)
self.add_spider_requests(spider_requests, url, new_depth)
def add_spider_requests(self, spider_requests, url, new_depth):
# if self.spider_queue.has_key(url):
# # TODO: allow for better rescan in future
# self.framework.log_warning('adding already scanned url [%s] for now' % (url))
for request in spider_requests:
queue_item = [None, request[0], request[1], request[2], request[3], request[4], url, 'P', new_depth]
rowid = self.Data.add_spider_queue_item(
self.write_cursor, queue_item
)
queue_item[0] = rowid
self.queueDataModel.append_data([queue_item])
def do_populateExistingSpiderData(self):
### print('starting populating responses')
self.qlock.lock()
try:
rows = []
for row in self.Data.get_spider_queue_items(self.read_cursor, 'P'):
rows.append([m or '' for m in row])
self.queueDataModel.append_data(rows)
rows = []
for row in self.Data.read_spider_pending_responses(self.read_cursor, 'P'):
response_id = int(row[SpiderPendingResponsesTable.RESPONSE_ID])
request_type = str(row[SpiderPendingResponsesTable.REQUEST_TYPE])
depth = int(row[SpiderPendingResponsesTable.DEPTH])
status = str(row[SpiderPendingResponsesTable.STATUS])
rows.append([response_id, request_type, depth, status])
self.pendingResponsesDataModel.append_data(rows)
rows = []
for row in self.Data.read_spider_pending_analysis(self.read_cursor):
analysis_id = int(row[SpiderPendingAnalysisTable.ID])
analysis_type = str(row[SpiderPendingAnalysisTable.ANALYSIS_TYPE])
content = str(row[SpiderPendingAnalysisTable.CONTENT])
url = str(row[SpiderPendingAnalysisTable.URL])
depth = int(row[SpiderPendingAnalysisTable.DEPTH])
data_item = [analysis_id, analysis_type, content, url, depth]
rows.append(data_item)
self.pendingAnalysisDataModel.append_data(rows)
finally:
self.qlock.unlock()
### print('finished populating responses')
def do_clearSpiderQueue(self):
self.qlock.lock()
try:
self.Data.clear_spider_queue(self.write_cursor)
self.queueDataModel.clearModel()
finally:
self.qlock.unlock()
def do_clearSpiderPendingResponses(self):
self.qlock.lock()
try:
self.Data.clear_spider_pending_responses(self.write_cursor)
self.pendingResponsesDataModel.clearModel()
finally:
self.qlock.unlock()
def do_resetSpiderPendingResponses(self):
self.qlock.lock()
try:
self.Data.reset_spider_pending_responses(self.write_cursor)
self.pendingResponsesDataModel.clearModel()
finally:
self.qlock.unlock()
def calculate_spider_requests(self, url, headers, body, content_type, depth):
requests = []
requests.extend(self.process_http_headers(url, headers))
content_type, charset = self.contentExtractor.parseContentType(content_type)
base_type = self.contentExtractor.getBaseType(content_type)
if 'html' == base_type:
requests.extend(self.process_html_data(url, body, charset))
else:
# TODO: implement other types
self.framework.log_warning('skipping unsupported type for request for [%s]: %s' % (url, content_type))
return self.filter_spider_requests(requests, depth)
def calculate_spider_requests_from_analysis(self, analysis_item):
requests = []
#--> print('ANALYSIS ->', analysis_item)
analysis_id, analysis_type, content, url, depth = analysis_item
depth = int(depth)
if 'url' == analysis_type:
self.append_url_link_request(requests, url, content)
elif 'html' == analysis_type:
requests.extend(self.process_html_data(url, content, 'utf-8')) # TODO: could extract ?
elif 'response_id' == analysis_type:
response_id = int(content)
self.add_pending_spider_response_id(response_id, 'spider', depth+1)
self.add_pending_spider_response_id(response_id, 'render', depth+1)
else:
self.framework.log_warning('unhandled data_type: %s' % (data_type))
return self.filter_spider_requests(requests, depth)
def filter_spider_requests(self, requests, depth):
# make sure that request has not already been retrieved
filtered_requests = []
already_seen = {}
found_response_id = None
for request in requests:
print(('filter spider request', request))
method, base_url, query = request[0], request[1], request[2]
if query:
base_url += '?' + query
content_type = ''
if already_seen.get(base_url) == method:
found = True
else:
already_seen[base_url] = method
found = False
for row in self.Data.read_responses_by_url(self.read_cursor, base_url):
response_items = [m or '' for m in row]
if response_items[ResponsesTable.REQ_METHOD] == method:
content_type = str(response_items[ResponsesTable.RES_CONTENT_TYPE])
found = True
found_response_id = int(response_items[ResponsesTable.ID])
break
if not found:
# TODO: probably shouldn't go back to database for this ....
for row in self.Data.read_spider_queue_by_url(self.read_cursor, base_url):
response_items = [m or '' for m in row]
if response_items[SpiderQueueTable.STATUS] != 'D' and response_items[SpiderQueueTable.METHOD] == method:
found = True
break
if not found:
if self.spiderRules.should_include_url(base_url):
filtered_requests.append(request)
elif found_response_id:
if not self.Data.spider_pending_response_exists(self.read_cursor2, found_response_id, 'spider'):
self.add_pending_spider_response_id(found_response_id, 'spider', depth)
# TODO: fix this hack
if 'html' in content_type.lower():
if not self.Data.spider_pending_response_exists(self.read_cursor2, found_response_id, 'render'):
self.add_pending_spider_response_id(found_response_id, 'render', depth)
return filtered_requests
def process_http_headers(self, url, headers):
links = []
for line in headers.splitlines():
m = self.re_location_header.match(line)
if m:
links.append(m.group(1))
continue
m = self.re_content_location_header.match(line)
if m:
links.append(m.group(1))
continue
if 0 == len(links):
return []
requests = []
for link in links:
self.append_url_link_request(requests, url, link)
return requests
def append_url_link_request(self, requests, base_url, link):
resolved_url = urlparse.urljoin(base_url, link)
if not self.scopeController.isUrlInScope(resolved_url, base_url):
return
splitted = urlparse.urlsplit(resolved_url)
if splitted.scheme in ('http', 'https'):
# TODO: check query for unique parameters
url = urlparse.urlunsplit((splitted.scheme, splitted.netloc, splitted.path, '', ''))
requests.append(('GET', url, splitted.query, '', ''))
def process_html_data(self, url, body, charset):
requests = []
results = self.htmlExtractor.process(body, url, charset, None)
# TODO: check fingerprints here ?
for link in results.links:
# TODO: all links should be already resolved ?
self.append_url_link_request(requests, url, link)
for form in results.forms:
link = form.action
if not self.scopeController.isUrlInScope(link, url):
continue
splitted = urlparse.urlsplit(link)
if splitted.scheme in ('http', 'https'):
# TODO: check query and form for unique parameters
base_url = urlparse.urlunsplit((splitted.scheme, splitted.netloc, splitted.path, '', ''))
form_data = self.get_form_data(form)
requests.append((form.method.upper(), base_url, splitted.query, form.enctype, form_data))
return requests
def get_form_data(self, form):
body_io = StringIO()
# TODO: spidering does not support uploading file data
# create all parameters as named/value parameters, if multipart enctype, generate that when sending
for i in range(0, len(form.inputs)):
name, value = self.get_form_input_value(form.inputs[i])
if 0 != i:
body_io.write('&')
if value is not None:
body_io.write('%s=%s' % (urllib.parse.quote(name), urllib.parse.quote(value)))
else:
body_io.write('%s' % (urllib.parse.quote(name)))
return body_io.getvalue()
def get_form_input_value(self, input):
# TODO: consider values without names?
if self.spiderConfig.use_data_bank:
name = input.name
value, fill_type = self.formFiller.populate_form_value(input.name, input.Id, input.value, input.Type, input.Class, input.required, input.maxlength, input.accept, input.label)
if fill_type in ('Username', 'Password') and not self.spiderConfig.submit_user_name_password:
# use whatever came in
value = input.value
else:
name = input.name
if not input.value:
value = self.formFiller.populate_generic_value(input.name, input.Id, input.value, input.Type, input.Class, input.required, input.maxlength, input.accept, input.label)
else:
value = input.value
return name, value
def make_spider_request_content(self, data_item):
spider_id, method, url, query_params, encoding_type, form_params, referer, status, depth = data_item
headers = {}
if referer:
headers['Referer'] = referer
body = ''
target_url = url
if query_params:
target_url += '?' + query_params
if 'POST' == method:
headers['Content-Type'] = encoding_type
if 'application/x-www-form-urlencoded' == encoding_type:
body = form_params
else:
# TODO: implement
raise Exception('implement me multiparm')
return method, target_url, headers, body
def process_page_html_content(self, html, url, depth):
self.qlock_analysis.lock()
try:
analysis_item = ['html', html, url, depth]
self.analysis_queue.append(analysis_item)
finally:
self.qlock_analysis.unlock()
QTimer.singleShot(10, self, SIGNAL('addPendingAnalysis()'))
def process_page_url_link(self, url, link, depth):
self.qlock_analysis.lock()
try:
analysis_item = ['url', link, url, depth]
self.analysis_queue.append(analysis_item)
finally:
self.qlock_analysis.unlock()
QTimer.singleShot(10, self, SIGNAL('addPendingAnalysis()'))
def process_page_response_id(self, response_id, depth):
self.qlock_analysis.lock()
try:
analysis_item = ['response_id', str(response_id), '', depth]
self.analysis_queue.append(analysis_item)
finally:
self.qlock_analysis.unlock()
QTimer.singleShot(10, self, SIGNAL('addPendingAnalysis()'))
def do_addPendingAnalysis(self):
self.qlock_analysis.lock()
try:
self.qlock.lock()
try:
rows = []
while (len(self.analysis_queue) > 0):
analysis_item = self.analysis_queue.popleft()
data_item = [None, analysis_item[0], analysis_item[1], analysis_item[2], analysis_item[3]]
rowid = self.Data.add_spider_pending_analysis(self.write_cursor, data_item)
data_item[0] = rowid
rows.append(data_item)
self.pendingAnalysisDataModel.append_data(rows)
finally:
self.qlock.unlock()
finally:
self.qlock_analysis.unlock()
class RequesterTab(QObject):
def __init__(self, framework, mainWindow):
QObject.__init__(self, mainWindow)
self.framework = framework
self.mainWindow = mainWindow
self.mainWindow.requesterSendButton.clicked.connect(
self.requester_send_button_clicked)
self.mainWindow.bulkRequestPushButton.clicked.connect(
self.requester_bulk_request_button_clicked)
self.mainWindow.requesterHistoryClearButton.clicked.connect(
self.requester_history_clear_button_clicked)
self.mainWindow.reqTabWidget.currentChanged.connect(
self.handle_tab_currentChanged)
self.mainWindow.requesterSequenceCheckBox.stateChanged.connect(
self.handle_requesterSequenceCheckBox_stateChanged)
self.mainWindow.bulkRequestSequenceCheckBox.stateChanged.connect(
self.handle_bulkRequestSequenceCheckBox_stateChanged)
self.mainWindow.sequenceRunnerRunButton.clicked.connect(
self.handle_sequenceRunnerRunButton_clicked)
self.pending_request = None
self.pending_bulk_requests = None
self.pending_sequence_requests = None
self.re_request = re.compile(
r'^(\S+)\s+((?:https?://(?:\S+\.)+\w+(?::\d+)?)?/.*)\s+HTTP/\d+\.\d+\s*$',
re.I)
self.re_request_cookie = re.compile(r'^Cookie:\s*(\S+)', re.I | re.M)
self.re_replacement = re.compile(r'\$\{(\w+)\}')
self.framework.subscribe_populate_requester_response_id(
self.requester_populate_response_id)
self.framework.subscribe_populate_bulk_requester_responses(
self.bulk_requester_populate_responses)
self.framework.subscribe_sequences_changed(self.fill_sequences)
self.setup_requester_tab()
self.Data = None
self.cursor = None
self.framework.subscribe_database_events(self.db_attach,
self.db_detach)
def db_attach(self):
self.Data = self.framework.getDB()
self.cursor = self.Data.allocate_thread_cursor()
self.fill_requesters()
def db_detach(self):
self.close_cursor()
self.Data = None
def close_cursor(self):
if self.cursor and self.Data:
self.cursor.close()
self.Data.release_thread_cursor(self.cursor)
self.cursor = None
def setup_requester_tab(self):
self.historyRequestResponse = RequestResponseWidget(
self.framework, self.mainWindow.requesterHistoryTabWidget,
self.mainWindow.requesterHistorySearchResultsPlaceholder, self)
self.requesterHistoryDataModel = ResponsesDataModel.ResponsesDataModel(
self.framework, self)
self.mainWindow.requesterHistoryTreeView.setModel(
self.requesterHistoryDataModel)
self.mainWindow.requesterHistoryTreeView.activated.connect(
self.fill_history_request_response)
self.mainWindow.requesterHistoryTreeView.clicked.connect(
self.fill_history_request_response)
self.mainWindow.requesterHistoryTreeView.doubleClicked.connect(
self.requester_history_item_double_clicked)
self.historyResponsesContextMenu = ResponsesContextMenuWidget(
self.framework, self.requesterHistoryDataModel,
self.mainWindow.requesterHistoryTreeView, self)
self.historyResponsesContextMenu.set_currentChanged_callback(
self.fill_history_request_response)
self.sequenceRunnerRequestResponse = RequestResponseWidget(
self.framework, self.mainWindow.sequenceRunnerTabWidget,
self.mainWindow.sequenceRunnerSearchResultsPlaceholder, self)
self.sequenceRunnerDataModel = ResponsesDataModel.ResponsesDataModel(
self.framework, self)
self.mainWindow.sequenceRunnerTreeView.setModel(
self.sequenceRunnerDataModel)
self.mainWindow.sequenceRunnerTreeView.activated.connect(
self.fill_sequence_runner_request_response)
self.mainWindow.sequenceRunnerTreeView.clicked.connect(
self.fill_sequence_runner_request_response)
self.mainWindow.sequenceRunnerTreeView.doubleClicked.connect(
self.requester_sequence_runner_item_double_clicked)
self.sequence_runnerResponsesContextMenu = ResponsesContextMenuWidget(
self.framework, self.sequenceRunnerDataModel,
self.mainWindow.sequenceRunnerTreeView, self)
self.sequence_runnerResponsesContextMenu.set_currentChanged_callback(
self.fill_sequence_runner_request_response)
self.miniResponseRenderWidget = MiniResponseRenderWidget(
self.framework, self.mainWindow.reqRespTabWidget, True, self)
self.scopeController = self.framework.getScopeController()
def requester_history_item_double_clicked(self, index):
Id = interface.index_to_id(self.requesterHistoryDataModel, index)
if Id:
dialog = RequestResponseDetailDialog(self.framework, Id,
self.mainWindow)
dialog.show()
dialog.exec_()
def fill_history_request_response(self, index):
Id = interface.index_to_id(self.requesterHistoryDataModel, index)
if Id:
self.historyRequestResponse.fill(Id)
def requester_sequence_runner_item_double_clicked(self, index):
Id = interface.index_to_id(self.sequenceRunnerDataModel, index)
if Id:
dialog = RequestResponseDetailDialog(self.framework, Id,
self.mainWindow)
dialog.show()
dialog.exec_()
def fill_sequence_runner_request_response(self, index):
Id = interface.index_to_id(self.sequenceRunnerDataModel, index)
if Id:
self.sequenceRunnerRequestResponse.fill(Id)
def fill_requesters(self):
# requesters
self.requesterHistoryDataModel.clearModel()
history_items = []
for row in self.Data.get_all_requester_history(self.cursor):
response_item = interface.data_row_to_response_items(row)
history_items.append(response_item)
self.requesterHistoryDataModel.append_data(history_items)
self.fill_sequences()
self.mainWindow.requesterUrlEdit.setText(
self.framework.get_raft_config_value('requesterUrlEdit'))
self.mainWindow.bulkRequestUrlListEdit.setPlainText(
self.framework.get_raft_config_value('bulkRequestUrlListEdit'))
def fill_sequences(self):
self.fill_sequences_combo_box(
self.mainWindow.requesterSequenceComboBox)
self.fill_sequences_combo_box(
self.mainWindow.bulkRequestSequenceComboBox)
self.fill_sequences_combo_box(
self.mainWindow.sequenceRunnerSequenceComboBox)
def requester_populate_response_id(self, Id):
row = self.Data.read_responses_by_id(self.cursor, Id)
if not row:
return
responseItems = interface.data_row_to_response_items(row)
method, url, template_text = self.generate_template_for_response_item(
responseItems)
self.set_combo_box_text(self.mainWindow.requesterRequestMethod,
method.upper())
self.mainWindow.requesterUrlEdit.setText(url)
self.mainWindow.requesterTemplateEdit.setPlainText(template_text)
def bulk_requester_populate_responses(self, id_list):
url_list = []
first = True
for Id in id_list:
row = self.Data.read_responses_by_id(self.cursor, Id)
if not row:
continue
responseItems = interface.data_row_to_response_items(row)
url = responseItems[ResponsesTable.URL]
if url not in url_list:
url_list.append(url)
if first:
method, url, template_text = self.generate_template_for_response_item(
responseItems)
self.set_combo_box_text(self.mainWindow.bulkRequestMethodEdit,
method.upper())
self.mainWindow.bulkRequestTemplateEdit.setPlainText(
template_text)
first = False
self.mainWindow.bulkRequestUrlListEdit.setPlainText(
'\n'.join(url_list))
def generate_template_for_response_item(self, responseItems):
url = responseItems[ResponsesTable.URL]
reqHeaders = str(responseItems[ResponsesTable.REQ_HEADERS], 'utf-8',
'ignore')
reqData = str(responseItems[ResponsesTable.REQ_DATA], 'utf-8',
'ignore')
method = responseItems[ResponsesTable.REQ_METHOD]
splitted = urlparse.urlsplit(url)
useragent = self.framework.useragent()
has_cookie = False
template = StringIO()
template.write('${method} ${request_uri} HTTP/1.1\n')
first = True
for line in reqHeaders.splitlines():
if not line:
break
if first and self.re_request.match(line):
first = False
continue
if ':' in line:
name, value = [v.strip() for v in line.split(':', 1)]
lname = name.lower()
if 'host' == lname:
if splitted.hostname and value == splitted.hostname:
template.write('Host: ${host}\n')
continue
elif 'user-agent' == lname:
if useragent == value:
template.write('User-Agent: ${user_agent}\n')
continue
template.write(line)
template.write('\n')
template.write('\n')
template.write(reqData)
return method, url, template.getvalue()
def set_combo_box_text(self, comboBox, selectedText):
index = comboBox.findText(selectedText)
if -1 == index:
comboBox.addItem(selectedText)
index = comboBox.findText(selectedText)
comboBox.setCurrentIndex(index)
def handle_requesterSequenceCheckBox_stateChanged(self, state):
self.mainWindow.requesterSequenceComboBox.setEnabled(
self.mainWindow.requesterSequenceCheckBox.isChecked())
def handle_bulkRequestSequenceCheckBox_stateChanged(self, state):
self.mainWindow.bulkRequestSequenceComboBox.setEnabled(
self.mainWindow.bulkRequestSequenceCheckBox.isChecked())
def handle_tab_currentChanged(self, index):
# TODO: must this hard-coded ?
if 0 == index:
self.fill_sequences_combo_box(
self.mainWindow.requesterSequenceComboBox)
elif 1 == index:
self.fill_sequences_combo_box(
self.mainWindow.bulkRequestSequenceComboBox)
elif 2 == index:
self.fill_sequences_combo_box(
self.mainWindow.sequenceRunnerSequenceComboBox)
def requester_send_button_clicked(self):
""" Make a request from the Request tab """
if 'Cancel' == self.mainWindow.requesterSendButton.text(
) and self.pending_request is not None:
self.pending_request.cancel()
self.pending_request = None
self.mainWindow.requesterSendButton.setText('Send')
return
qurl = QUrl.fromUserInput(self.mainWindow.requesterUrlEdit.text())
url = qurl.toEncoded().data().decode('utf-8')
self.mainWindow.requesterUrlEdit.setText(url)
self.framework.set_raft_config_value('requesterUrlEdit', url)
templateText = str(self.mainWindow.requesterTemplateEdit.toPlainText())
method = str(self.mainWindow.requesterRequestMethod.currentText())
use_global_cookie_jar = self.mainWindow.requesterUseGlobalCookieJar.isChecked(
)
replacements = self.build_replacements(method, url)
(method, url, headers,
body) = self.process_template(url, templateText, replacements)
sequenceId = None
if self.mainWindow.requesterSequenceCheckBox.isChecked():
sequenceId = str(
self.mainWindow.requesterSequenceComboBox.itemData(
self.mainWindow.requesterSequenceComboBox.currentIndex()))
self.requestRunner = RequestRunner(self.framework, self)
if use_global_cookie_jar:
self.requesterCookieJar = self.framework.get_global_cookie_jar()
else:
self.requesterCookieJar = InMemoryCookieJar(self.framework, self)
self.requestRunner.setup(self.requester_response_received,
self.requesterCookieJar, sequenceId)
self.pending_request = self.requestRunner.queue_request(
method, url, headers, body)
self.mainWindow.requesterSendButton.setText('Cancel')
self.miniResponseRenderWidget.clear_response_render()
def requester_response_received(self, response_id, context):
if 0 != response_id:
row = self.Data.read_responses_by_id(self.cursor, response_id)
if row:
response_item = interface.data_row_to_response_items(row)
self.Data.insert_requester_history(self.cursor, response_id)
self.requesterHistoryDataModel.append_data([response_item])
url = response_item[ResponsesTable.URL]
req_headers = response_item[ResponsesTable.REQ_HEADERS]
req_body = response_item[ResponsesTable.REQ_DATA]
res_headers = response_item[ResponsesTable.RES_HEADERS]
res_body = response_item[ResponsesTable.RES_DATA]
res_content_type = response_item[
ResponsesTable.RES_CONTENT_TYPE]
self.miniResponseRenderWidget.populate_response_content(
url, req_headers, req_body, res_headers, res_body,
res_content_type)
self.mainWindow.requesterSendButton.setText('Send')
self.pending_request = None
def requester_bulk_request_button_clicked(self):
if 'Cancel' == self.mainWindow.bulkRequestPushButton.text(
) and self.pending_bulk_requests is not None:
self.cancel_bulk_requests = True
for context, pending_request in self.pending_bulk_requests.items():
pending_request.cancel()
self.pending_bulk_requests = None
self.mainWindow.bulkRequestPushButton.setText('Send')
self.mainWindow.bulkRequestProgressBar.setValue(0)
return
if self.pending_bulk_requests is None:
self.pending_bulk_requests = {}
method = str(self.mainWindow.bulkRequestMethodEdit.currentText())
templateText = str(
self.mainWindow.bulkRequestTemplateEdit.toPlainText())
template_url = str(self.mainWindow.bulkRequestUrlEdit.text())
url_list = str(self.mainWindow.bulkRequestUrlListEdit.toPlainText())
self.framework.set_raft_config_value('bulkRequestUrlListEdit',
url_list)
request_urls = url_list.splitlines()
self.mainWindow.bulkRequestProgressBar.setValue(0)
self.mainWindow.bulkRequestProgressBar.setMaximum(len(request_urls))
sequenceId = None
if self.mainWindow.bulkRequestSequenceCheckBox.isChecked():
sequenceId = str(
self.mainWindow.bulkRequestSequenceComboBox.itemData(
self.mainWindow.bulkRequestSequenceComboBox.currentIndex())
)
first = True
self.cancel_bulk_requests = False
for request_url in request_urls:
if self.cancel_bulk_requests:
break
request_url = request_url.strip()
if request_url:
context = uuid.uuid4().hex
# TODO: move this hack
if '$' in template_url:
replacements = self.build_replacements(method, request_url)
url = self.re_replacement.sub(
lambda m: replacements.get(m.group(1)), template_url)
else:
url = request_url
if not self.scopeController.isUrlInScope(url, url):
self.framework.log_warning(
'skipping out of scope URL: %s' % (url))
self.mainWindow.bulkRequestProgressBar.setValue(
self.mainWindow.bulkRequestProgressBar.value() + 1)
continue
use_global_cookie_jar = self.mainWindow.bulkRequestUseGlobalCookieJar.isChecked(
)
replacements = self.build_replacements(method, url)
(method, url, headers,
body) = self.process_template(url, templateText, replacements)
if first:
self.mainWindow.bulkRequestPushButton.setText('Cancel')
if use_global_cookie_jar:
self.bulkRequesterCookieJar = self.framework.get_global_cookie_jar(
)
else:
self.bulkRequesterCookieJar = InMemoryCookieJar(
self.framework, self)
self.bulk_requestRunner = RequestRunner(
self.framework, self)
self.bulk_requestRunner.setup(
self.requester_bulk_response_received,
self.bulkRequesterCookieJar, sequenceId)
first = False
self.pending_bulk_requests[
context] = self.bulk_requestRunner.queue_request(
method, url, headers, body, context)
def requester_bulk_response_received(self, response_id, context):
self.mainWindow.bulkRequestProgressBar.setValue(
self.mainWindow.bulkRequestProgressBar.value() + 1)
context = str(context)
if self.pending_bulk_requests is not None:
try:
self.pending_bulk_requests.pop(context)
except KeyError as e:
pass
if 0 != response_id:
row = self.Data.read_responses_by_id(self.cursor, response_id)
if row:
response_item = interface.data_row_to_response_items(row)
self.Data.insert_requester_history(self.cursor, response_id)
self.requesterHistoryDataModel.append_data([response_item])
finished = False
if self.pending_bulk_requests is None or len(
self.pending_bulk_requests) == 0:
self.mainWindow.bulkRequestProgressBar.setValue(
self.mainWindow.bulkRequestProgressBar.maximum())
finished = True
elif self.mainWindow.bulkRequestProgressBar.value(
) == self.mainWindow.bulkRequestProgressBar.maximum():
finished = True
if finished:
self.mainWindow.bulkRequestPushButton.setText('Send')
def handle_sequenceRunnerRunButton_clicked(self):
""" Run a sequence """
if 'Cancel' == self.mainWindow.sequenceRunnerRunButton.text(
) and self.pending_sequence_requests is not None:
self.cancel_sequence_requests = True
for context, pending_request in self.pending_sequence_requests.items(
):
pending_request.cancel()
self.pending_sequence_requests = None
self.mainWindow.sequenceRunnerButton.setText('Send')
self.mainWindow.sequenceRunnerButton.setValue(0)
return
self.sequenceRunnerDataModel.clearModel()
sequenceId = str(
self.mainWindow.sequenceRunnerSequenceComboBox.itemData(
self.mainWindow.sequenceRunnerSequenceComboBox.currentIndex()))
use_global_cookie_jar = self.mainWindow.sequenceRunnerUseGlobalCookieJar.isChecked(
)
if use_global_cookie_jar:
self.sequenceRunnerCookieJar = self.framework.get_global_cookie_jar(
)
else:
self.sequenceRunnerCookieJar = InMemoryCookieJar(
self.framework, self)
self.sequence_requestRunner = RequestRunner(self.framework, self)
self.sequence_requestRunner.setup(
self.sequence_runner_response_received,
self.sequenceRunnerCookieJar, sequenceId)
self.pending_sequence_requests = self.sequence_requestRunner.run_sequence(
)
self.mainWindow.sequenceRunnerRunButton.setText('Cancel')
def sequence_runner_response_received(self, response_id, context):
context = str(context)
if self.pending_sequence_requests is not None:
try:
self.pending_sequence_requests.pop(context)
except KeyError as e:
print((e))
pass
if 0 != response_id:
row = self.Data.read_responses_by_id(self.cursor, response_id)
if row:
response_item = interface.data_row_to_response_items(row)
self.sequenceRunnerDataModel.append_data([response_item])
if self.pending_sequence_requests is None or len(
self.pending_sequence_requests) == 0:
self.mainWindow.sequenceRunnerRunButton.setText('Send')
def requester_history_clear_button_clicked(self):
self.Data.clear_requester_history(self.cursor)
self.requesterHistoryDataModel.clearModel()
def fill_sequences_combo_box(self, comboBox):
selectedText = comboBox.currentText()
comboBox.clear()
for row in self.Data.get_all_sequences(self.cursor):
sequenceItem = [m or '' for m in row]
name = str(sequenceItem[1])
Id = str(sequenceItem[0])
item = comboBox.addItem(name, Id)
if selectedText:
index = comboBox.findText(selectedText)
if index != -1:
comboBox.setCurrentIndex(index)
def build_replacements(self, method, url):
replacements = {}
splitted = urlparse.urlsplit(url)
replacements['method'] = method.upper()
replacements['url'] = url
replacements['scheme'] = splitted.scheme or ''
replacements['netloc'] = splitted.netloc or ''
replacements['host'] = splitted.hostname or ''
replacements['path'] = splitted.path or '/'
replacements['query'] = splitted.query or ''
replacements['fragment'] = splitted.fragment or ''
replacements['request_uri'] = urlparse.urlunsplit(
('', '', replacements['path'], replacements['query'], ''))
replacements['user_agent'] = self.framework.useragent()
return replacements
def process_template(self, url, template, replacements):
method, uri = '', ''
headers, body = '', ''
# TODO: this allows for missing entries -- is this good?
func = lambda m: replacements.get(m.group(1))
prev = 0
while True:
n = template.find('\n', prev)
if -1 == n:
break
if n > 0 and '\r' == template[n - 1]:
line = template[prev:n - 1]
else:
line = template[prev:n]
if 0 == len(line):
# end of headers
headers = template[0:n + 1]
body = template[n + 1:]
break
prev = n + 1
if not headers:
headers = template
body = ''
# TODO: could work from ordered dict to main order?
headers_dict = {}
first = True
for line in headers.splitlines():
if not line:
break
if '$' in line:
line = self.re_replacement.sub(func, line)
if first:
m = self.re_request.match(line)
if not m:
raise Exception(
'Invalid HTTP request: failed to match request line: %s'
% (line))
method = m.group(1)
uri = m.group(2)
first = False
continue
if ':' in line:
name, value = [v.strip() for v in line.split(':', 1)]
headers_dict[name] = value
if '$' in body:
body = self.re_replacement.sub(func, body)
url = urlparse.urljoin(url, uri)
return (method, url, headers_dict, body)
def requester_bulk_request_button_clicked(self):
if 'Cancel' == self.mainWindow.bulkRequestPushButton.text(
) and self.pending_bulk_requests is not None:
self.cancel_bulk_requests = True
for context, pending_request in self.pending_bulk_requests.items():
pending_request.cancel()
self.pending_bulk_requests = None
self.mainWindow.bulkRequestPushButton.setText('Send')
self.mainWindow.bulkRequestProgressBar.setValue(0)
return
if self.pending_bulk_requests is None:
self.pending_bulk_requests = {}
method = str(self.mainWindow.bulkRequestMethodEdit.currentText())
templateText = str(
self.mainWindow.bulkRequestTemplateEdit.toPlainText())
template_url = str(self.mainWindow.bulkRequestUrlEdit.text())
url_list = str(self.mainWindow.bulkRequestUrlListEdit.toPlainText())
self.framework.set_raft_config_value('bulkRequestUrlListEdit',
url_list)
request_urls = url_list.splitlines()
self.mainWindow.bulkRequestProgressBar.setValue(0)
self.mainWindow.bulkRequestProgressBar.setMaximum(len(request_urls))
sequenceId = None
if self.mainWindow.bulkRequestSequenceCheckBox.isChecked():
sequenceId = str(
self.mainWindow.bulkRequestSequenceComboBox.itemData(
self.mainWindow.bulkRequestSequenceComboBox.currentIndex())
)
first = True
self.cancel_bulk_requests = False
for request_url in request_urls:
if self.cancel_bulk_requests:
break
request_url = request_url.strip()
if request_url:
context = uuid.uuid4().hex
# TODO: move this hack
if '$' in template_url:
replacements = self.build_replacements(method, request_url)
url = self.re_replacement.sub(
lambda m: replacements.get(m.group(1)), template_url)
else:
url = request_url
if not self.scopeController.isUrlInScope(url, url):
self.framework.log_warning(
'skipping out of scope URL: %s' % (url))
self.mainWindow.bulkRequestProgressBar.setValue(
self.mainWindow.bulkRequestProgressBar.value() + 1)
continue
use_global_cookie_jar = self.mainWindow.bulkRequestUseGlobalCookieJar.isChecked(
)
replacements = self.build_replacements(method, url)
(method, url, headers,
body) = self.process_template(url, templateText, replacements)
if first:
self.mainWindow.bulkRequestPushButton.setText('Cancel')
if use_global_cookie_jar:
self.bulkRequesterCookieJar = self.framework.get_global_cookie_jar(
)
else:
self.bulkRequesterCookieJar = InMemoryCookieJar(
self.framework, self)
self.bulk_requestRunner = RequestRunner(
self.framework, self)
self.bulk_requestRunner.setup(
self.requester_bulk_response_received,
self.bulkRequesterCookieJar, sequenceId)
first = False
self.pending_bulk_requests[
context] = self.bulk_requestRunner.queue_request(
method, url, headers, body, context)
class SpiderThread(QThread):
def __init__(self,
framework,
queueDataModel,
pendingResponsesDataModel,
pendingAnalysisDataModel,
internalStateDataModel,
parent=None):
QThread.__init__(self, parent)
self.framework = framework
self.queueDataModel = queueDataModel
self.pendingResponsesDataModel = pendingResponsesDataModel
self.pendingAnalysisDataModel = pendingAnalysisDataModel
self.internalStateDataModel = internalStateDataModel
self.qlock = QMutex()
self.qlock_analysis = QMutex()
QObject.connect(self, SIGNAL('quit()'), self.quitHandler)
QObject.connect(self, SIGNAL('started()'), self.startedHandler)
def do_setup(self):
self.spider_items = {}
self.spider_outstanding_requests = {}
self.analysis_queue = deque()
self.scopeController = self.framework.getScopeController()
self.contentExtractor = self.framework.getContentExtractor()
self.htmlExtractor = self.contentExtractor.getExtractor('html')
self.spiderConfig = self.framework.getSpiderConfig()
self.spiderRules = SpiderRules(self.framework, self)
self.formFiller = FormFiller(self.framework, self)
self.re_location_header = re.compile(r'^Location:\s*(.+)$', re.I)
self.re_content_location_header = re.compile(
r'^Content-Location:\s*(.+)$', re.I)
self.Data = None
self.read_cursor = None
self.read_cursor2 = None
self.write_cursor = None
self.keep_spidering = False
def db_attach(self):
self.Data = self.framework.getDB()
self.read_cursor = self.Data.allocate_thread_cursor()
self.read_cursor2 = self.Data.allocate_thread_cursor()
self.write_cursor = self.Data.allocate_thread_cursor()
self.populateExistingSpiderData()
def db_detach(self):
self.close_cursor()
self.Data = None
def close_cursor(self):
if self.write_cursor and self.Data:
self.write_cursor.close()
self.Data.release_thread_cursor(self.write_cursor)
self.write_cursor = None
if self.read_cursor2 and self.Data:
self.read_cursor2.close()
self.Data.release_thread_cursor(self.read_cursor2)
self.read_cursor2 = None
if self.read_cursor and self.Data:
self.read_cursor.close()
self.Data.release_thread_cursor(self.read_cursor)
self.read_cursor = None
def run(self):
QObject.connect(self, SIGNAL('populateExistingSpiderData()'),
self.do_populateExistingSpiderData,
Qt.DirectConnection)
QObject.connect(self, SIGNAL('clearSpiderQueue()'),
self.do_clearSpiderQueue, Qt.DirectConnection)
QObject.connect(self, SIGNAL('clearSpiderPendingResponses()'),
self.do_clearSpiderPendingResponses,
Qt.DirectConnection)
QObject.connect(self, SIGNAL('resetSpiderPendingResponses()'),
self.do_resetSpiderPendingResponses,
Qt.DirectConnection)
QObject.connect(self, SIGNAL('startSpidering()'),
self.do_startSpidering, Qt.DirectConnection)
QObject.connect(self, SIGNAL('stopSpidering()'), self.do_stopSpidering,
Qt.DirectConnection)
QObject.connect(self, SIGNAL('spiderItemFinished()'),
self.do_spiderItemFinished, Qt.DirectConnection)
QObject.connect(self, SIGNAL('generateSpiderValues()'),
self.do_generateSpiderValues, Qt.DirectConnection)
QObject.connect(self, SIGNAL('sendNextSpiderRequest()'),
self.do_sendNextSpiderRequest, Qt.DirectConnection)
QObject.connect(self, SIGNAL('addPendingAnalysis()'),
self.do_addPendingAnalysis, Qt.DirectConnection)
self.exec_()
def quitHandler(self):
self.framework.debug_log('SpiderThread quit...')
self.close_cursor()
self.exit(0)
def startedHandler(self):
self.framework.debug_log('SpiderThread started...')
self.do_setup()
self.framework.subscribe_database_events(self.db_attach,
self.db_detach)
self.framework.subscribe_populate_spider_response_id(
self.do_populate_spider_response_id)
self.framework.subscribe_populate_spider_response_list(
self.do_populate_spider_response_list)
def populateExistingSpiderData(self):
QTimer.singleShot(50, self, SIGNAL('populateExistingSpiderData()'))
def clearSpiderQueue(self):
QTimer.singleShot(50, self, SIGNAL('clearSpiderQueue()'))
def clearSpiderPendingResponses(self):
QTimer.singleShot(50, self, SIGNAL('clearSpiderPendingResponses()'))
def resetSpiderPendingResponses(self):
QTimer.singleShot(50, self, SIGNAL('resetSpiderPendingResponses()'))
def startSpidering(self, spider_callback, sequence_id, cookieJar):
print('startSpidering')
self.spider_callback = spider_callback
if sequence_id and sequence_id > 0:
self.sequence_id = sequence_id
else:
self.sequence_id = None
self.cookieJar = cookieJar
QTimer.singleShot(100, self, SIGNAL('startSpidering()'))
def stopSpidering(self):
print('stopSpidering')
self.keep_spidering = False
QTimer.singleShot(50, self, SIGNAL('stopSpidering()'))
def spiderItemFinished(self, response_id):
###-> print('*****1')
self.qlock.lock()
###-> print('*****2')
try:
###-> print('*****3')
self.Data.update_spider_pending_response_id(
self.write_cursor, 'C', response_id, 'render')
###-> print('*****4')
finally:
self.qlock.unlock()
###-> print('*****5')
self.handle_spider_available()
def do_startSpidering(self):
print('do_startSpidering')
# TODO: decide about global cookies ?
self.requestRunner = RequestRunner(self.framework, self)
self.requestRunner.setup(self.network_response_received,
self.cookieJar, self.sequence_id)
self.keep_spidering = True
self.renderer_available = False
self.handle_spider_available()
def do_spiderItemFinished(self):
pass
###-> print('****6')
def handle_spider_available(self):
if self.keep_spidering:
QTimer.singleShot(10, self, SIGNAL('generateSpiderValues()'))
do_send = False
self.qlock.lock()
try:
if len(self.spider_outstanding_requests) == 0:
do_send = True
finally:
self.qlock.unlock()
if do_send:
QTimer.singleShot(10, self, SIGNAL('sendNextSpiderRequest()'))
self.dispatch_next_render_item()
def do_populate_spider_response_id(self, response_id):
self.qlock.lock()
try:
self.add_pending_spider_response(response_id, 0)
finally:
self.qlock.unlock()
QTimer.singleShot(50, self, SIGNAL('generateSpiderValues()'))
def do_populate_spider_response_list(self, id_list):
self.qlock.lock()
try:
for response_id in id_list:
self.add_pending_spider_response(int(response_id), 0)
self.add_pending_spider_response(int(response_id), 0)
finally:
self.qlock.unlock()
QTimer.singleShot(50, self, SIGNAL('generateSpiderValues()'))
def add_pending_spider_response(self, response_id, depth):
row = self.Data.read_responses_by_id(self.read_cursor, response_id)
if not row:
self.framework.log_warning('missing response id: %s' %
(response_id))
return
response_items = [m or '' for m in row]
content_type = str(response_items[ResponsesTable.RES_CONTENT_TYPE])
content_type, charset = self.contentExtractor.parseContentType(
content_type)
base_type = self.contentExtractor.getBaseType(content_type)
if 'html' == base_type:
self.add_pending_spider_response_id(response_id, 'spider', depth)
self.add_pending_spider_response_id(response_id, 'render', depth)
else:
# TODO: implement other render types
self.add_pending_spider_response_id(response_id, 'spider', depth)
self.framework.log_warning(
'skipping unsupported type for render analysis for [%s]: %s' %
(response_id, content_type))
def add_pending_spider_response_id(self, response_id, request_type, depth):
data_item = [response_id, request_type, depth, 'P']
if self.Data.add_spider_pending_response_id(self.write_cursor,
data_item):
self.pendingResponsesDataModel.append_data([data_item])
def do_generateSpiderValues(self):
self.generate_from_pending_responses()
self.generate_from_pending_analysis()
def generate_from_pending_responses(self):
self.qlock.lock()
keep_looping = True
try:
putback_rows = []
print('generate spider values from pending responses')
while keep_looping:
data_item = self.pendingResponsesDataModel.popleft_data()
###-> print(data_item)
if data_item:
response_id, request_type, depth, status = data_item
else:
keep_looping = False
if data_item is not None:
if 'spider' == request_type:
self.generate_spider_values(response_id, depth)
# remove from database
self.Data.update_spider_pending_response_id(
self.write_cursor, 'C', response_id, request_type)
else:
# put back
putback_rows.append(data_item)
self.pendingResponsesDataModel.append_data(putback_rows)
except Exception as error:
self.framework.report_exception(error)
finally:
self.qlock.unlock()
def generate_from_pending_analysis(self):
keep_looping = True
self.qlock.lock()
try:
print('generate spider values from analysis queues')
while self.keep_spidering and keep_looping:
data_item = self.pendingAnalysisDataModel.popleft_data()
###-> print(data_item)
if data_item:
analysis_id, analysis_type, content, url, depth = data_item
if depth:
new_depth = int(depth) + 1
else:
new_depth = 1
if new_depth < self.spiderConfig.max_link_depth:
spider_requests = self.calculate_spider_requests_from_analysis(
data_item)
self.add_spider_requests(spider_requests, url,
new_depth)
self.Data.delete_spider_pending_analysis(
self.write_cursor, analysis_id)
else:
keep_looping = False
except Exception as error:
self.framework.report_exception(error)
finally:
self.qlock.unlock()
def dispatch_next_render_item(self):
###-> print('****7')
if self.keep_spidering:
###-> print('****8')
render_item = self.get_next_render_item()
if render_item:
self.renderer_available = False
QObject.emit(
self.spider_callback,
SIGNAL('spiderItemAvailable(int, QString, QUrl, int)'),
render_item[0], render_item[1], render_item[2],
render_item[3])
else:
self.renderer_available = True
def do_sendNextSpiderRequest(self):
while True:
spider_request = self.get_next_spider_request()
###-> print(spider_request)
if not spider_request:
return
if spider_request:
method, url, headers, body, context = spider_request
if self.scopeController.isUrlInScope(url, url):
self.requestRunner.queue_request(method, url, headers,
body, context)
return
else:
self.framework.log_warning('SKIPPING out of scope: [%s]' %
(url))
self.qlock.lock()
try:
data_item = self.spider_outstanding_requests.pop(
context)
self.Data.update_spider_queue_item_status(
self.write_cursor,
int(data_item[SpiderQueueTable.ID]), 'C')
except KeyError:
pass
finally:
self.qlock.unlock()
def network_response_received(self, response_id, context):
data_item = None
context = str(context)
if context:
self.qlock.lock()
try:
if context not in self.spider_outstanding_requests:
self.framework.log_warning(
'*** missing spider request for [%s]' % (context))
else:
data_item = self.spider_outstanding_requests.pop(context)
self.Data.update_spider_queue_item_status(
self.write_cursor, int(data_item[SpiderQueueTable.ID]),
'C')
self.add_pending_spider_response_id(
response_id, 'spider',
int(data_item[SpiderQueueTable.DEPTH]))
self.add_pending_spider_response_id(
response_id, 'render',
int(data_item[SpiderQueueTable.DEPTH]))
finally:
self.qlock.unlock()
if self.keep_spidering:
QTimer.singleShot(50, self, SIGNAL('generateSpiderValues()'))
QTimer.singleShot(50, self, SIGNAL('sendNextSpiderRequest()'))
# TODO: checking concurrency issues
if self.renderer_available:
self.dispatch_next_render_item()
def get_next_render_item(self):
render_item = None
self.qlock.lock()
keep_looping = True
try:
putback_rows = []
while keep_looping:
data_item = self.pendingResponsesDataModel.popleft_data()
###-> print(data_item)
if data_item:
response_id, request_type, depth, status = data_item
else:
keep_looping = False
if data_item is not None:
if 'render' == request_type:
row = self.Data.read_responses_by_id(
self.read_cursor, response_id)
if not row:
self.framework.log_warning(
'missing response id: %s' % (response_id))
continue
response_items = [m or '' for m in row]
qurl = QUrl.fromEncoded(
response_items[ResponsesTable.URL])
dataContent = str(
response_items[ResponsesTable.RES_DATA])
render_item = (response_id, dataContent, qurl, depth)
keep_looping = False
else:
# put back
putback_rows.append(data_item)
self.pendingResponsesDataModel.appendleft_data(putback_rows)
except Exception as error:
self.framework.report_exception(error)
finally:
self.qlock.unlock()
###--> print('next render item', render_item)
return render_item
def get_next_spider_request(self):
self.qlock.lock()
spider_request = None
try:
data_item = self.queueDataModel.popleft_data()
if data_item:
method, target_url, headers, body = self.make_spider_request_content(
data_item)
context = uuid.uuid4().hex
self.spider_outstanding_requests[context] = data_item
spider_request = (method, target_url, headers, body, context)
finally:
self.qlock.unlock()
###--> print('next spider_request item', spider_request)
return spider_request
def do_stopSpidering(self):
self.keep_spidering = False
print(('do_stopSpidering', self, self.keep_spidering))
def generate_spider_values(self, response_id, depth):
new_depth = depth + 1
if new_depth >= self.spiderConfig.max_link_depth:
return
row = self.Data.read_responses_by_id(self.read_cursor, response_id)
if not row:
self.framework.log_warning('missing response id: %s' %
(response_id))
return
response_items = [m or '' for m in row]
url = str(response_items[ResponsesTable.URL])
response_headers = str(response_items[ResponsesTable.RES_HEADERS])
response_body = str(response_items[ResponsesTable.RES_DATA])
content_type = str(response_items[ResponsesTable.RES_CONTENT_TYPE])
spider_requests = self.calculate_spider_requests(
url, response_headers, response_body, content_type, new_depth)
self.add_spider_requests(spider_requests, url, new_depth)
def add_spider_requests(self, spider_requests, url, new_depth):
# if self.spider_queue.has_key(url):
# # TODO: allow for better rescan in future
# self.framework.log_warning('adding already scanned url [%s] for now' % (url))
for request in spider_requests:
queue_item = [
None, request[0], request[1], request[2], request[3],
request[4], url, 'P', new_depth
]
rowid = self.Data.add_spider_queue_item(self.write_cursor,
queue_item)
queue_item[0] = rowid
self.queueDataModel.append_data([queue_item])
def do_populateExistingSpiderData(self):
### print('starting populating responses')
self.qlock.lock()
try:
rows = []
for row in self.Data.get_spider_queue_items(self.read_cursor, 'P'):
rows.append([m or '' for m in row])
self.queueDataModel.append_data(rows)
rows = []
for row in self.Data.read_spider_pending_responses(
self.read_cursor, 'P'):
response_id = int(row[SpiderPendingResponsesTable.RESPONSE_ID])
request_type = str(
row[SpiderPendingResponsesTable.REQUEST_TYPE])
depth = int(row[SpiderPendingResponsesTable.DEPTH])
status = str(row[SpiderPendingResponsesTable.STATUS])
rows.append([response_id, request_type, depth, status])
self.pendingResponsesDataModel.append_data(rows)
rows = []
for row in self.Data.read_spider_pending_analysis(
self.read_cursor):
analysis_id = int(row[SpiderPendingAnalysisTable.ID])
analysis_type = str(
row[SpiderPendingAnalysisTable.ANALYSIS_TYPE])
content = str(row[SpiderPendingAnalysisTable.CONTENT])
url = str(row[SpiderPendingAnalysisTable.URL])
depth = int(row[SpiderPendingAnalysisTable.DEPTH])
data_item = [analysis_id, analysis_type, content, url, depth]
rows.append(data_item)
self.pendingAnalysisDataModel.append_data(rows)
finally:
self.qlock.unlock()
### print('finished populating responses')
def do_clearSpiderQueue(self):
self.qlock.lock()
try:
self.Data.clear_spider_queue(self.write_cursor)
self.queueDataModel.clearModel()
finally:
self.qlock.unlock()
def do_clearSpiderPendingResponses(self):
self.qlock.lock()
try:
self.Data.clear_spider_pending_responses(self.write_cursor)
self.pendingResponsesDataModel.clearModel()
finally:
self.qlock.unlock()
def do_resetSpiderPendingResponses(self):
self.qlock.lock()
try:
self.Data.reset_spider_pending_responses(self.write_cursor)
self.pendingResponsesDataModel.clearModel()
finally:
self.qlock.unlock()
def calculate_spider_requests(self, url, headers, body, content_type,
depth):
requests = []
requests.extend(self.process_http_headers(url, headers))
content_type, charset = self.contentExtractor.parseContentType(
content_type)
base_type = self.contentExtractor.getBaseType(content_type)
if 'html' == base_type:
requests.extend(self.process_html_data(url, body, charset))
else:
# TODO: implement other types
self.framework.log_warning(
'skipping unsupported type for request for [%s]: %s' %
(url, content_type))
return self.filter_spider_requests(requests, depth)
def calculate_spider_requests_from_analysis(self, analysis_item):
requests = []
#--> print('ANALYSIS ->', analysis_item)
analysis_id, analysis_type, content, url, depth = analysis_item
depth = int(depth)
if 'url' == analysis_type:
self.append_url_link_request(requests, url, content)
elif 'html' == analysis_type:
requests.extend(self.process_html_data(
url, content, 'utf-8')) # TODO: could extract ?
elif 'response_id' == analysis_type:
response_id = int(content)
self.add_pending_spider_response_id(response_id, 'spider',
depth + 1)
self.add_pending_spider_response_id(response_id, 'render',
depth + 1)
else:
self.framework.log_warning('unhandled data_type: %s' % (data_type))
return self.filter_spider_requests(requests, depth)
def filter_spider_requests(self, requests, depth):
# make sure that request has not already been retrieved
filtered_requests = []
already_seen = {}
found_response_id = None
for request in requests:
print(('filter spider request', request))
method, base_url, query = request[0], request[1], request[2]
if query:
base_url += '?' + query
content_type = ''
if already_seen.get(base_url) == method:
found = True
else:
already_seen[base_url] = method
found = False
for row in self.Data.read_responses_by_url(
self.read_cursor, base_url):
response_items = [m or '' for m in row]
if response_items[ResponsesTable.REQ_METHOD] == method:
content_type = str(
response_items[ResponsesTable.RES_CONTENT_TYPE])
found = True
found_response_id = int(
response_items[ResponsesTable.ID])
break
if not found:
# TODO: probably shouldn't go back to database for this ....
for row in self.Data.read_spider_queue_by_url(
self.read_cursor, base_url):
response_items = [m or '' for m in row]
if response_items[
SpiderQueueTable.STATUS] != 'D' and response_items[
SpiderQueueTable.METHOD] == method:
found = True
break
if not found:
if self.spiderRules.should_include_url(base_url):
filtered_requests.append(request)
elif found_response_id:
if not self.Data.spider_pending_response_exists(
self.read_cursor2, found_response_id, 'spider'):
self.add_pending_spider_response_id(
found_response_id, 'spider', depth)
# TODO: fix this hack
if 'html' in content_type.lower():
if not self.Data.spider_pending_response_exists(
self.read_cursor2, found_response_id, 'render'):
self.add_pending_spider_response_id(
found_response_id, 'render', depth)
return filtered_requests
def process_http_headers(self, url, headers):
links = []
for line in headers.splitlines():
m = self.re_location_header.match(line)
if m:
links.append(m.group(1))
continue
m = self.re_content_location_header.match(line)
if m:
links.append(m.group(1))
continue
if 0 == len(links):
return []
requests = []
for link in links:
self.append_url_link_request(requests, url, link)
return requests
def append_url_link_request(self, requests, base_url, link):
resolved_url = urlparse.urljoin(base_url, link)
if not self.scopeController.isUrlInScope(resolved_url, base_url):
return
splitted = urlparse.urlsplit(resolved_url)
if splitted.scheme in ('http', 'https'):
# TODO: check query for unique parameters
url = urlparse.urlunsplit(
(splitted.scheme, splitted.netloc, splitted.path, '', ''))
requests.append(('GET', url, splitted.query, '', ''))
def process_html_data(self, url, body, charset):
requests = []
results = self.htmlExtractor.process(body, url, charset, None)
# TODO: check fingerprints here ?
for link in results.links:
# TODO: all links should be already resolved ?
self.append_url_link_request(requests, url, link)
for form in results.forms:
link = form.action
if not self.scopeController.isUrlInScope(link, url):
continue
splitted = urlparse.urlsplit(link)
if splitted.scheme in ('http', 'https'):
# TODO: check query and form for unique parameters
base_url = urlparse.urlunsplit(
(splitted.scheme, splitted.netloc, splitted.path, '', ''))
form_data = self.get_form_data(form)
requests.append((form.method.upper(), base_url, splitted.query,
form.enctype, form_data))
return requests
def get_form_data(self, form):
body_io = StringIO()
# TODO: spidering does not support uploading file data
# create all parameters as named/value parameters, if multipart enctype, generate that when sending
for i in range(0, len(form.inputs)):
name, value = self.get_form_input_value(form.inputs[i])
if 0 != i:
body_io.write('&')
if value is not None:
body_io.write(
'%s=%s' %
(urllib.parse.quote(name), urllib.parse.quote(value)))
else:
body_io.write('%s' % (urllib.parse.quote(name)))
return body_io.getvalue()
def get_form_input_value(self, input):
# TODO: consider values without names?
if self.spiderConfig.use_data_bank:
name = input.name
value, fill_type = self.formFiller.populate_form_value(
input.name, input.Id, input.value, input.Type, input.Class,
input.required, input.maxlength, input.accept, input.label)
if fill_type in (
'Username', 'Password'
) and not self.spiderConfig.submit_user_name_password:
# use whatever came in
value = input.value
else:
name = input.name
if not input.value:
value = self.formFiller.populate_generic_value(
input.name, input.Id, input.value, input.Type, input.Class,
input.required, input.maxlength, input.accept, input.label)
else:
value = input.value
return name, value
def make_spider_request_content(self, data_item):
spider_id, method, url, query_params, encoding_type, form_params, referer, status, depth = data_item
headers = {}
if referer:
headers['Referer'] = referer
body = ''
target_url = url
if query_params:
target_url += '?' + query_params
if 'POST' == method:
headers['Content-Type'] = encoding_type
if 'application/x-www-form-urlencoded' == encoding_type:
body = form_params
else:
# TODO: implement
raise Exception('implement me multiparm')
return method, target_url, headers, body
def process_page_html_content(self, html, url, depth):
self.qlock_analysis.lock()
try:
analysis_item = ['html', html, url, depth]
self.analysis_queue.append(analysis_item)
finally:
self.qlock_analysis.unlock()
QTimer.singleShot(10, self, SIGNAL('addPendingAnalysis()'))
def process_page_url_link(self, url, link, depth):
self.qlock_analysis.lock()
try:
analysis_item = ['url', link, url, depth]
self.analysis_queue.append(analysis_item)
finally:
self.qlock_analysis.unlock()
QTimer.singleShot(10, self, SIGNAL('addPendingAnalysis()'))
def process_page_response_id(self, response_id, depth):
self.qlock_analysis.lock()
try:
analysis_item = ['response_id', str(response_id), '', depth]
self.analysis_queue.append(analysis_item)
finally:
self.qlock_analysis.unlock()
QTimer.singleShot(10, self, SIGNAL('addPendingAnalysis()'))
def do_addPendingAnalysis(self):
self.qlock_analysis.lock()
try:
self.qlock.lock()
try:
rows = []
while (len(self.analysis_queue) > 0):
analysis_item = self.analysis_queue.popleft()
data_item = [
None, analysis_item[0], analysis_item[1],
analysis_item[2], analysis_item[3]
]
rowid = self.Data.add_spider_pending_analysis(
self.write_cursor, data_item)
data_item[0] = rowid
rows.append(data_item)
self.pendingAnalysisDataModel.append_data(rows)
finally:
self.qlock.unlock()
finally:
self.qlock_analysis.unlock()
def start_fuzzing_clicked(self):
""" Start the fuzzing attack """
if 'Cancel' == self.mainWindow.wfStdStartButton.text() and self.pending_fuzz_requests is not None:
self.cancel_fuzz_requests = True
for context, pending_request in self.pending_fuzz_requests.items():
pending_request.cancel()
self.pending_fuzz_requests = None
self.mainWindow.wfStdStartButton.setText('Start Attack')
self.mainWindow.fuzzerStandardProgressBar.setValue(0)
return
self.pending_fuzz_requests = {}
url = str(self.mainWindow.wfStdUrlEdit.text())
templateText = str(self.mainWindow.wfStdEdit.toPlainText())
method = str(self.mainWindow.stdFuzzerReqMethod.currentText())
self.save_standard_configuration()
replacements = self.build_replacements(method, url)
sequenceId = None
if self.mainWindow.wfStdPreChk.isChecked():
sequenceId = self.mainWindow.wfStdPreBox.itemData(self.mainWindow.wfStdPreBox.currentIndex())
postSequenceId = None
if self.mainWindow.wfStdPostChk.isChecked():
postSequenceId = self.mainWindow.wfStdPostBox.itemData(self.mainWindow.wfStdPostBox.currentIndex())
# Fuzzing stuff
payload_mapping = self.create_payload_map()
# print(payload_mapping)
self.create_functions()
template_definition = TemplateDefinition(templateText)
template_items = template_definition.template_items
### print(template_items)
parameter_names = template_definition.parameter_names
self.global_ns = self.local_ns = {}
scriptLoader = ScriptLoader()
errors = []
fuzz_payloads = {}
for name, payload_info in payload_mapping.items():
if name in parameter_names:
payload_type, payload_value, payload_file = payload_info
if 'fuzz' == payload_type:
filename = payload_value
values = self.Attacks.read_data(filename)
fuzz_payloads[name] = values
elif 'dynamic' == payload_type:
target = payload_file
# TODO: should this come from saved file or current Scintilla values (?)
script_env = scriptLoader.load_from_file(os.path.join(self.functions_dir, target), self.global_ns, self.local_ns)
expression = payload_value
if not expression.endswith('()'):
expression += '()'
eval_result = eval(expression, self.global_ns, self.local_ns)
fuzz_payloads[name] = [str(v) for v in eval_result]
elif 'static' == payload_type:
pass
elif 'none' == payload_type:
# unconfigured payload
errors.append(name)
test_slots = []
counters = []
tests_count = []
total_tests = 1
for name, payload_info in payload_mapping.items():
if name in parameter_names:
payload_type, payload_value, payload_file = payload_info
if 'static' == payload_type:
# static payload value
payloads = [payload_value]
elif 'fuzz' == payload_type:
payloads = fuzz_payloads[name]
elif 'dynamic' == payload_type:
payloads = fuzz_payloads[name]
total_tests *= len(payloads)
test_slots.append((name, payloads))
counters.append(0)
tests_count.append(len(payloads))
position_end = len(counters) - 1
position = position_end
self.miniResponseRenderWidget.clear_response_render()
self.mainWindow.fuzzerStandardProgressBar.setValue(0)
self.mainWindow.fuzzerStandardProgressBar.setMaximum(total_tests)
finished = False
first = True
while not finished:
data = {}
for j in range(0, len(test_slots)):
name, payloads = test_slots[j]
data[name] = payloads[counters[j]]
template_io = StringIO()
self.apply_template_parameters(template_io, data, template_items)
templateText = template_io.getvalue()
context = uuid.uuid4().hex
# print('%s%s%s' % ('-'*32, request, '-'*32))
use_global_cookie_jar = self.mainWindow.webFuzzerUseGlobalCookieJar.isChecked()
(method, url, headers, body) = self.process_template(url, templateText, replacements)
if first:
self.mainWindow.wfStdStartButton.setText('Cancel')
if use_global_cookie_jar:
self.fuzzRequesterCookieJar = self.framework.get_global_cookie_jar()
else:
self.fuzzRequesterCookieJar = InMemoryCookieJar(self.framework, self)
self.requestRunner = RequestRunner(self.framework, self)
self.requestRunner.setup(self.fuzzer_response_received, self.fuzzRequesterCookieJar, sequenceId, postSequenceId)
first = False
self.pending_fuzz_requests[context] = self.requestRunner.queue_request(method, url, headers, body, context)
# increment to next test
counters[position] = (counters[position] + 1) % (tests_count[position])
while position >= 0 and counters[position] == 0:
position -= 1
counters[position] = (counters[position] + 1) % (tests_count[position])
if position == -1:
finished = True
else:
position = position_end
class WebFuzzerTab(QObject):
def __init__(self, framework, mainWindow):
QObject.__init__(self, mainWindow)
self.framework = framework
self.mainWindow = mainWindow
self.mainWindow.wfStdPreChk.stateChanged.connect(self.handle_wfStdPreChk_stateChanged)
self.mainWindow.wfStdPostChk.stateChanged.connect(self.handle_wfStdPostChk_stateChanged)
self.mainWindow.wfTempSeqChk.stateChanged.connect(self.handle_wfTempSeqChk_stateChanged)
# Handle the toggling of payload mappings in the config tab
self.mainWindow.wfPay1FuzzRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay1StaticRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay1DynamicRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay2FuzzRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay2StaticRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay2DynamicRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay3FuzzRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay3StaticRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay3DynamicRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay4FuzzRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay4StaticRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay4DynamicRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay5FuzzRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay5StaticRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.wfPay5DynamicRadio.toggled.connect(self.handle_payload_toggled)
self.mainWindow.fuzzerHistoryClearButton.clicked.connect(self.fuzzer_history_clear_button_clicked)
# inserted to initially fill the sequences box.
# ToDo: Need to do this better
self.mainWindow.mainTabWidget.currentChanged.connect(self.handle_mainTabWidget_currentChanged)
self.mainWindow.webFuzzTab.currentChanged.connect(self.handle_webFuzzTab_currentChanged)
self.mainWindow.stdFuzzTab.currentChanged.connect(self.handle_stdFuzzTab_currentChanged)
# self.mainWindow.webFuzzTab.currentChanged.connect(self.fill_payloads)
self.mainWindow.wfStdAddButton.clicked.connect(self.insert_payload_marker)
self.mainWindow.wfStdStartButton.clicked.connect(self.start_fuzzing_clicked)
self.mainWindow.wfDataDictonaryAddButton.clicked.connect(self.handle_wfDataDictonaryAddButton_clicked)
self.framework.subscribe_populate_webfuzzer_response_id(self.webfuzzer_populate_response_id)
self.framework.subscribe_sequences_changed(self.fill_sequences)
self.mainWindow.wfFunctionsComboBox.activated.connect(self.fill_function_edit)
self.mainWindow.wfFunctionsSaveButton.clicked.connect(self.save_function_file)
self.mainWindow.wfFunctionsDeleteButton.clicked.connect(self.del_function_file)
self.miniResponseRenderWidget = MiniResponseRenderWidget(self.framework, self.mainWindow.stdFuzzResultsTabWidget, True, self)
self.re_request = re.compile(r'^(\S+)\s+((?:https?://(?:\S+\.)+\w+(?::\d+)?)?/.*)\s+HTTP/\d+\.\d+\s*$', re.I)
self.re_request_cookie = re.compile(r'^Cookie:\s*(\S+)', re.I|re.M)
self.re_replacement = re.compile(r'\$\{(\w+)\}')
self.setup_fuzzer_tab()
self.setup_functions_tab()
self.functions_dir = os.path.join(self.framework.get_data_dir(), 'functions')
self.Attacks = Payloads.Payloads(self.framework)
self.Attacks.list_files()
# Fill the payloads combo boxes on init
self.fill_payloads()
self.pending_fuzz_requests = None
# Fill the functions combo box on init
self.fill_function_combo_box()
self.mainWindow.wfDataDictionaryDataTable.setColumnCount(2)
self.mainWindow.wfDataDictionaryDataTable.setHorizontalHeaderLabels(['Replacement', 'Value'])
self.Data = None
self.cursor = None
self.framework.subscribe_database_events(self.db_attach, self.db_detach)
def db_attach(self):
self.Data = self.framework.getDB()
self.cursor = self.Data.allocate_thread_cursor()
self.fill_fuzzers()
self.fill_standard_edits()
self.fill_config_edits()
self.restore_data_dictionary()
def db_detach(self):
self.close_cursor()
self.Data = None
def close_cursor(self):
if self.cursor and self.Data:
self.cursor.close()
self.Data.release_thread_cursor(self.cursor)
self.cursor = None
def setup_fuzzer_tab(self):
self.fuzzerHistoryDataModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
self.mainWindow.fuzzerHistoryTreeView.setModel(self.fuzzerHistoryDataModel)
self.mainWindow.fuzzerHistoryTreeView.doubleClicked.connect(self.fuzzer_history_item_double_clicked)
self.mainWindow.fuzzerHistoryTreeView.clicked.connect(self.handle_fuzzer_history_clicked)
self.mainWindow.fuzzerHistoryTreeView.activated.connect(self.handle_fuzzer_history_clicked)
self.responsesContextMenu = ResponsesContextMenuWidget(self.framework, self.fuzzerHistoryDataModel, self.mainWindow.fuzzerHistoryTreeView, self)
self.responsesContextMenu.set_currentChanged_callback(self.fill_fuzzer_history)
def setup_functions_tab(self):
self.functionsLayout = self.mainWindow.wfFunctionsEditPlaceholder.layout()
if not self.functionsLayout:
self.functionsLayout = QVBoxLayout(self.mainWindow.wfFunctionsEditPlaceholder)
self.functionsEditScintilla = Qsci.QsciScintilla(self.mainWindow.wfFunctionsEditPlaceholder)
ScintillaHelpers.SetScintillaProperties(self.framework, self.functionsEditScintilla, 'python')
self.functionsEditScintilla.setAutoIndent(True)
self.functionsLayout.addWidget(self.functionsEditScintilla)
self.framework.subscribe_zoom_in(self.edit_function_zoom_in)
self.framework.subscribe_zoom_out(self.edit_function_zoom_out)
def edit_function_zoom_in(self):
self.functionsEditScintilla.zoomIn()
def edit_function_zoom_out(self):
self.functionsEditScintilla.zoomOut()
def fill_fuzzers(self):
history_items = []
for row in self.Data.get_all_fuzzer_history(self.cursor):
response_item = [m or '' for m in row]
history_items.append(response_item)
self.fuzzerHistoryDataModel.append_data(history_items)
self.fill_sequences()
def fill_standard_edits(self):
self.mainWindow.wfStdUrlEdit.setText(self.framework.get_raft_config_value('WebFuzzer.Standard.RequestUrl'))
self.mainWindow.wfStdEdit.document().setHtml(self.framework.get_raft_config_value('WebFuzzer.Standard.TemplateHtml'))
index = self.mainWindow.stdFuzzerReqMethod.findText(self.framework.get_raft_config_value('WebFuzzer.Standard.Method'))
if index != -1:
self.mainWindow.stdFuzzerReqMethod.setCurrentIndex(index)
self.mainWindow.wfStdPreChk.setChecked(self.framework.get_raft_config_value('WebFuzzer.Standard.PreSequenceEnabled', bool))
index = self.mainWindow.wfStdPreBox.findText(self.framework.get_raft_config_value('WebFuzzer.Standard.PreSequenceId'))
if index != -1:
self.mainWindow.wfStdPreBox.setCurrentIndex(index)
self.mainWindow.wfStdPostChk.setChecked(self.framework.get_raft_config_value('WebFuzzer.Standard.PostSequenceEnabled', bool))
index = self.mainWindow.wfStdPostBox.findText(self.framework.get_raft_config_value('WebFuzzer.Standard.PostSequenceId'))
if index != -1:
self.mainWindow.wfStdPostBox.setCurrentIndex(index)
def fill_config_edits(self):
self.fill_config_edit_item('Payload1', self.mainWindow.wfPay1FuzzRadio, self.mainWindow.wfPay1PayloadBox, self.mainWindow.wfPay1StaticRadio, self.mainWindow.wfPay1DynamicRadio, self.mainWindow.wfPay1StaticEdit)
self.fill_config_edit_item('Payload2', self.mainWindow.wfPay2FuzzRadio, self.mainWindow.wfPay2PayloadBox, self.mainWindow.wfPay2StaticRadio, self.mainWindow.wfPay2DynamicRadio, self.mainWindow.wfPay2StaticEdit)
self.fill_config_edit_item('Payload3', self.mainWindow.wfPay3FuzzRadio, self.mainWindow.wfPay3PayloadBox, self.mainWindow.wfPay3StaticRadio, self.mainWindow.wfPay3DynamicRadio, self.mainWindow.wfPay3StaticEdit)
self.fill_config_edit_item('Payload4', self.mainWindow.wfPay4FuzzRadio, self.mainWindow.wfPay4PayloadBox, self.mainWindow.wfPay4StaticRadio, self.mainWindow.wfPay4DynamicRadio, self.mainWindow.wfPay4StaticEdit)
self.fill_config_edit_item('Payload5', self.mainWindow.wfPay5FuzzRadio, self.mainWindow.wfPay5PayloadBox, self.mainWindow.wfPay5StaticRadio, self.mainWindow.wfPay5DynamicRadio, self.mainWindow.wfPay5StaticEdit)
def fill_config_edit_item(self, payload_item, fuzzRadio, payloadBox, staticRadio, dynamicRadio, staticEdit):
fuzzRadio.setChecked(self.framework.get_raft_config_value('WebFuzzer.Config.{0}FuzzSelected'.format(payload_item), bool))
index = payloadBox.findText(self.framework.get_raft_config_value('WebFuzzer.Config.{0}FuzzPayload'.format(payload_item)))
if index != -1:
payloadBox.setCurrentIndex(index)
staticRadio.setChecked(self.framework.get_raft_config_value('WebFuzzer.Config.{0}StaticSelected'.format(payload_item), bool))
dynamicRadio.setChecked(self.framework.get_raft_config_value('WebFuzzer.Config.{0}DynamicSelected'.format(payload_item), bool))
staticEdit.setText(self.framework.get_raft_config_value('WebFuzzer.Config.{0}StaticEdit'.format(payload_item)))
def fuzzer_history_item_double_clicked(self, index):
Id = interface.index_to_id(self.fuzzerHistoryDataModel, index)
if Id:
dialog = RequestResponseDetailDialog(self.framework, Id, self.mainWindow)
dialog.show()
dialog.exec_()
def handle_mainTabWidget_currentChanged(self):
self.save_configuration_values()
def handle_stdFuzzTab_currentChanged(self):
self.save_standard_configuration()
def handle_webFuzzTab_currentChanged(self):
self.save_configuration_values()
def fill_sequences(self):
self.fill_sequences_combo_box(self.mainWindow.wfStdPreBox)
self.fill_sequences_combo_box(self.mainWindow.wfStdPostBox)
self.fill_sequences_combo_box(self.mainWindow.wfStdBox)
def fill_sequences_combo_box(self, comboBox):
selectedText = comboBox.currentText()
comboBox.clear()
for row in self.Data.get_all_sequences(self.cursor):
sequenceItem = [m or '' for m in row]
name = str(sequenceItem[1])
Id = str(sequenceItem[0])
item = comboBox.addItem(name, Id)
if selectedText:
index = comboBox.findText(selectedText)
if index != -1:
comboBox.setCurrentIndex(index)
def fill_payloads(self):
self.fill_payload_combo_box(self.mainWindow.wfPay1PayloadBox)
self.fill_payload_combo_box(self.mainWindow.wfPay2PayloadBox)
self.fill_payload_combo_box(self.mainWindow.wfPay3PayloadBox)
self.fill_payload_combo_box(self.mainWindow.wfPay4PayloadBox)
self.fill_payload_combo_box(self.mainWindow.wfPay5PayloadBox)
def fill_payload_combo_box(self, comboBox):
selectedText = comboBox.currentText()
comboBox.clear()
payloads = self.Attacks.list_files()
for item in payloads:
if item.startswith("."):
pass
else:
comboBox.addItem(item)
def fill_payload_combo_box_function(self, comboBox):
selectedText = comboBox.currentText()
comboBox.clear()
functions = self.Attacks.list_function_files()
for item in functions:
if item.startswith("."):
pass
else:
comboBox.addItem(item)
def fill_function_combo_box(self):
comboBox = self.mainWindow.wfFunctionsComboBox
comboBox.clear()
functions = self.Attacks.list_function_files()
for item in functions:
if item.startswith("."):
pass
else:
comboBox.addItem(item)
def fill_function_edit(self):
filename = self.mainWindow.wfFunctionsComboBox.currentText()
func = self.Attacks.read_function(filename)
# Clear the Scintilla widget
self.functionsEditScintilla.clear()
for line in func:
self.functionsEditScintilla.append(line.decode("utf8"))
def save_function_file(self):
function_file = self.mainWindow.wfFunctionsComboBox.currentText()
content = self.functionsEditScintilla.text()
self.Attacks.save_function(function_file, content)
def del_function_file(self):
# Gets the current name of the file selected in the combobox
filename = self.mainWindow.wfFunctionsComboBox.currentText()
path = self.functions_dir
message = "Are you sure you want to delete: {0}".format(filename)
response = ConfirmDialog.display_confirm_dialog(self.mainWindow, message)
if response == True:
os.remove(os.path.join(path,filename))
self.fill_function_combo_box()
# Clear the items from the scintilla widget
#ToDo: This should be the default data for the function
self.functionsEditScintilla.clear()
def create_payload_map(self):
# create payload map from configuration tab
payload_mapping = {}
payload_config_items = (
("payload_1",
"fuzz", self.mainWindow.wfPay1FuzzRadio, self.mainWindow.wfPay1PayloadBox,
"static", self.mainWindow.wfPay1StaticRadio, self.mainWindow.wfPay1StaticEdit,
"dynamic", self.mainWindow.wfPay1DynamicRadio, self.mainWindow.wfPay1StaticEdit,
),
("payload_2",
"fuzz", self.mainWindow.wfPay2FuzzRadio, self.mainWindow.wfPay2PayloadBox,
"static", self.mainWindow.wfPay2StaticRadio, self.mainWindow.wfPay2StaticEdit,
"dynamic", self.mainWindow.wfPay2DynamicRadio, self.mainWindow.wfPay2StaticEdit,
),
("payload_3",
"fuzz", self.mainWindow.wfPay3FuzzRadio, self.mainWindow.wfPay3PayloadBox,
"static", self.mainWindow.wfPay3StaticRadio, self.mainWindow.wfPay3StaticEdit,
"dynamic", self.mainWindow.wfPay3DynamicRadio, self.mainWindow.wfPay3StaticEdit,
),
("payload_4",
"fuzz", self.mainWindow.wfPay4FuzzRadio, self.mainWindow.wfPay4PayloadBox,
"static", self.mainWindow.wfPay4StaticRadio, self.mainWindow.wfPay4StaticEdit,
"dynamic", self.mainWindow.wfPay4DynamicRadio, self.mainWindow.wfPay4StaticEdit,
),
("payload_5",
"fuzz", self.mainWindow.wfPay5FuzzRadio, self.mainWindow.wfPay5PayloadBox,
"static", self.mainWindow.wfPay5StaticRadio, self.mainWindow.wfPay5StaticEdit,
"dynamic", self.mainWindow.wfPay5DynamicRadio, self.mainWindow.wfPay5StaticEdit,
),
)
# Determine active payloads and map them
for config_item in payload_config_items:
payload_item = config_item[0]
payload_mapping[payload_item] = ('none', '')
for offset in (1, 4, 7):
if config_item[offset+1].isChecked():
payload_type = config_item[offset]
# print((payload_type, offset))
if payload_type == "dynamic":
payload_mapping[payload_item] = (payload_type, str(config_item[offset+2].text()), config_item[3].currentText())
elif payload_type == "fuzz":
payload_mapping[payload_item] = (payload_type, str(config_item[offset+2].currentText()), '')
else:
payload_mapping[payload_item] = (payload_type, str(config_item[offset+2].text()), '')
break
return payload_mapping
def create_functions(self):
self.global_ns = self.local_ns = {}
functions = [
'''
import urllib.parse
import re
import random
def url_encode(input):
return urllib.parse.quote(input)
re_alert_mangler = re.compile(r'alert\([^(]+\)', re.I)
def randomize_alert_replace(m):
return 'alert(%d.%d)' % (random.randint(0,99999), random.randint(0,99999))
def randomize_alert(input):
return re_alert_mangler.sub(randomize_alert_replace, input)
'''
]
for func_str in functions:
compiled = compile(func_str, '<string>', 'exec')
exec(compiled, self.global_ns, self.local_ns)
def set_combo_box_text(self, comboBox, selectedText):
index = comboBox.findText(selectedText)
if -1 != index:
comboBox.setCurrentIndex(index)
else:
index = comboBox.addItem(selectedText)
comboBox.setCurrentIndex(index)
def handle_wfStdPreChk_stateChanged(self, state):
self.mainWindow.wfStdPreBox.setEnabled(self.mainWindow.wfStdPreChk.isChecked())
def handle_wfStdPostChk_stateChanged(self, state):
self.mainWindow.wfStdPostBox.setEnabled(self.mainWindow.wfStdPostChk.isChecked())
def handle_wfTempSeqChk_stateChanged(self, state):
self.mainWindow.wfStdBox.setEnabled(self.mainWindow.wfTempSeqChk.isChecked())
def handle_payload_toggled(self):
self.mainWindow.wfPay1PayloadBox.setEnabled(self.mainWindow.wfPay1FuzzRadio.isChecked() or self.mainWindow.wfPay1DynamicRadio.isChecked())
self.mainWindow.wfPay1StaticEdit.setEnabled(self.mainWindow.wfPay1StaticRadio.isChecked() or self.mainWindow.wfPay1DynamicRadio.isChecked())
# self.mainWindow.wfPay1PayloadBox.setEnabled(self.mainWindow.wfPay1DynamicRadio.isChecked()) or self.fill_payload_combo_box_function(self.mainWindow.wfPay1PayloadBox)
self.mainWindow.wfPay2PayloadBox.setEnabled(self.mainWindow.wfPay2FuzzRadio.isChecked() or self.mainWindow.wfPay2DynamicRadio.isChecked())
self.mainWindow.wfPay2StaticEdit.setEnabled(self.mainWindow.wfPay2StaticRadio.isChecked() or self.mainWindow.wfPay2DynamicRadio.isChecked())
self.mainWindow.wfPay3PayloadBox.setEnabled(self.mainWindow.wfPay3FuzzRadio.isChecked() or self.mainWindow.wfPay3DynamicRadio.isChecked())
self.mainWindow.wfPay3StaticEdit.setEnabled(self.mainWindow.wfPay3StaticRadio.isChecked() or self.mainWindow.wfPay3DynamicRadio.isChecked())
self.mainWindow.wfPay4PayloadBox.setEnabled(self.mainWindow.wfPay4FuzzRadio.isChecked() or self.mainWindow.wfPay4DynamicRadio.isChecked())
self.mainWindow.wfPay4StaticEdit.setEnabled(self.mainWindow.wfPay4StaticRadio.isChecked() or self.mainWindow.wfPay4DynamicRadio.isChecked())
self.mainWindow.wfPay5PayloadBox.setEnabled(self.mainWindow.wfPay5FuzzRadio.isChecked() or self.mainWindow.wfPay5DynamicRadio.isChecked())
self.mainWindow.wfPay5StaticEdit.setEnabled(self.mainWindow.wfPay5StaticRadio.isChecked() or self.mainWindow.wfPay5DynamicRadio.isChecked())
# Determine if fuzz or dynamic is selected and change combo box items
if self.mainWindow.wfPay1FuzzRadio.isChecked():
self.fill_payload_combo_box(self.mainWindow.wfPay1PayloadBox)
if self.mainWindow.wfPay1DynamicRadio.isChecked():
self.fill_payload_combo_box_function(self.mainWindow.wfPay1PayloadBox)
if self.mainWindow.wfPay2FuzzRadio.isChecked():
self.fill_payload_combo_box(self.mainWindow.wfPay2PayloadBox)
if self.mainWindow.wfPay2DynamicRadio.isChecked():
self.fill_payload_combo_box_function(self.mainWindow.wfPay2PayloadBox)
if self.mainWindow.wfPay3FuzzRadio.isChecked():
self.fill_payload_combo_box(self.mainWindow.wfPay3PayloadBox)
if self.mainWindow.wfPay3DynamicRadio.isChecked():
self.fill_payload_combo_box_function(self.mainWindow.wfPay3PayloadBox)
if self.mainWindow.wfPay4FuzzRadio.isChecked():
self.fill_payload_combo_box(self.mainWindow.wfPay4PayloadBox)
if self.mainWindow.wfPay4DynamicRadio.isChecked():
self.fill_payload_combo_box_function(self.mainWindow.wfPay4PayloadBox)
if self.mainWindow.wfPay5FuzzRadio.isChecked():
self.fill_payload_combo_box(self.mainWindow.wfPay5PayloadBox)
if self.mainWindow.wfPay5DynamicRadio.isChecked():
self.fill_payload_combo_box_function(self.mainWindow.wfPay5PayloadBox)
def handle_fuzzer_history_clicked(self):
index = self.mainWindow.fuzzerHistoryTreeView.currentIndex()
self.fill_fuzzer_history(index)
def fill_fuzzer_history(self, index):
Id = interface.index_to_id(self.fuzzerHistoryDataModel, index)
if Id:
row = self.Data.read_responses_by_id(self.cursor, Id)
if not row:
return
responseItems = interface.data_row_to_response_items(row)
url = responseItems[ResponsesTable.URL]
reqHeaders = responseItems[ResponsesTable.REQ_HEADERS]
reqData = responseItems[ResponsesTable.REQ_DATA]
resHeaders = responseItems[ResponsesTable.RES_HEADERS]
resData = responseItems[ResponsesTable.RES_DATA]
contentType = responseItems[ResponsesTable.RES_CONTENT_TYPE]
self.miniResponseRenderWidget.populate_response_content(url, reqHeaders, reqData, resHeaders, resData, contentType)
def webfuzzer_populate_response_id(self, Id):
self.clear_data_dictionary()
row = self.Data.read_responses_by_id(self.cursor, Id)
if not row:
return
responseItems = interface.data_row_to_response_items(row)
url = responseItems[ResponsesTable.URL]
reqHeaders = responseItems[ResponsesTable.REQ_HEADERS].decode('utf-8', 'ignore')
reqData = responseItems[ResponsesTable.REQ_DATA].decode('utf-8', 'ignore')
method = responseItems[ResponsesTable.REQ_METHOD]
splitted = urlparse.urlsplit(url)
# Create a new parsed object removing the scheme and netloc
base_url = urlparse.urlunsplit((splitted[0], splitted[1], splitted[2], '', ''))
req_loc = ("", "", "", splitted.query, splitted.fragment)
useragent = self.framework.useragent()
has_cookie = False
template = StringIO()
template.write('${method} ${request_uri}%s HTTP/1.1\n' % urlparse.urlunsplit(req_loc))
first = True
for line in reqHeaders.splitlines():
if not line:
break
if first and self.re_request.match(line):
first = False
continue
if ':' in line:
name, value = [v.strip() for v in line.split(':', 1)]
lname = name.lower()
if 'host' == lname:
if splitted.hostname and value == splitted.hostname:
template.write('Host: ${host}\n')
continue
elif 'user-agent' == lname:
if useragent == value:
template.write('User-Agent: ${user_agent}\n')
continue
template.write(line)
template.write('\n')
template.write('\n')
template.write(reqData)
self.set_combo_box_text(self.mainWindow.stdFuzzerReqMethod, method.upper())
self.mainWindow.wfStdUrlEdit.setText(base_url)
self.mainWindow.wfStdEdit.setPlainText(template.getvalue())
def insert_payload_marker(self):
""" Inserts a payload marker at current cursor position """
index = self.mainWindow.stdFuzzPayloadBox.currentIndex()
curPayload = str(self.mainWindow.stdFuzzPayloadBox.itemText(index))
currentText = self.mainWindow.wfStdEdit.textCursor().selectedText()
self.store_in_data_dictionary(curPayload, currentText)
self.mainWindow.wfStdEdit.textCursor().insertHtml("<font color='red'>${%s}</font>" % curPayload)
self.save_configuration_values()
def clear_data_dictionary(self):
tableWidget = self.mainWindow.wfDataDictionaryDataTable
while tableWidget.rowCount() > 0:
tableWidget.removeRow(0)
self.save_data_dictionary_to_config()
def store_in_data_dictionary(self, replacement, value):
# TODO: currently logic allows for duplicate values to appear
tableWidget = self.mainWindow.wfDataDictionaryDataTable
row = tableWidget.rowCount()
tableWidget.insertRow(row)
tableWidget.setItem(row, 0, QTableWidgetItem(replacement))
tableWidget.setItem(row, 1, QTableWidgetItem(value))
self.save_data_dictionary_to_config()
def save_data_dictionary_to_config(self):
ddict = self.get_values_from_data_dictionary()
dd_data = json.dumps(ddict)
self.framework.set_raft_config_value('WebFuzzer.Standard.DataDictionary', dd_data)
def restore_data_dictionary(self):
dd_data = self.framework.get_raft_config_value('WebFuzzer.Standard.DataDictionary', '')
if dd_data:
tableWidget = self.mainWindow.wfDataDictionaryDataTable
obj = json.loads(dd_data)
for name, value in obj.items():
row = tableWidget.rowCount()
tableWidget.insertRow(row)
tableWidget.setItem(row, 0, QTableWidgetItem(name))
tableWidget.setItem(row, 1, QTableWidgetItem(value))
def get_values_from_data_dictionary(self):
tableWidget = self.mainWindow.wfDataDictionaryDataTable
ddict = {}
for rindex in range(0, tableWidget.rowCount()):
name = tableWidget.item(rindex, 0).text()
value = tableWidget.item(rindex, 1).text()
ddict[name] = value
return ddict
def handle_wfDataDictonaryAddButton_clicked(self):
name = self.mainWindow.wfDataDictionaryDataName.text()
value = self.mainWindow.wfDataDictionaryDataValue.text()
self.store_in_data_dictionary(name, value)
def save_configuration_values(self):
self.save_standard_configuration()
self.save_config_configuration()
def save_standard_configuration(self):
url = str(self.mainWindow.wfStdUrlEdit.text())
templateHtml = str(self.mainWindow.wfStdEdit.document().toHtml())
method = str(self.mainWindow.stdFuzzerReqMethod.currentText())
self.framework.set_raft_config_value('WebFuzzer.Standard.RequestUrl', url)
self.framework.set_raft_config_value('WebFuzzer.Standard.TemplateHtml', templateHtml)
self.framework.set_raft_config_value('WebFuzzer.Standard.Method', method)
self.framework.set_raft_config_value('WebFuzzer.Standard.PreSequenceEnabled', self.mainWindow.wfStdPreChk.isChecked())
self.framework.set_raft_config_value('WebFuzzer.Standard.PreSequenceId', self.mainWindow.wfStdPreBox.itemData(self.mainWindow.wfStdPreBox.currentIndex()))
self.framework.set_raft_config_value('WebFuzzer.Standard.PostSequenceEnabled', self.mainWindow.wfStdPostChk.isChecked())
self.framework.set_raft_config_value('WebFuzzer.Standard.PostSequenceId', self.mainWindow.wfStdPostBox.itemData(self.mainWindow.wfStdPostBox.currentIndex()))
def save_config_configuration(self):
self.save_config_configuration_item('Payload1', self.mainWindow.wfPay1FuzzRadio, self.mainWindow.wfPay1PayloadBox, self.mainWindow.wfPay1StaticRadio, self.mainWindow.wfPay1DynamicRadio, self.mainWindow.wfPay1StaticEdit)
self.save_config_configuration_item('Payload2', self.mainWindow.wfPay2FuzzRadio, self.mainWindow.wfPay2PayloadBox, self.mainWindow.wfPay2StaticRadio, self.mainWindow.wfPay2DynamicRadio, self.mainWindow.wfPay2StaticEdit)
self.save_config_configuration_item('Payload3', self.mainWindow.wfPay3FuzzRadio, self.mainWindow.wfPay3PayloadBox, self.mainWindow.wfPay3StaticRadio, self.mainWindow.wfPay3DynamicRadio, self.mainWindow.wfPay3StaticEdit)
self.save_config_configuration_item('Payload4', self.mainWindow.wfPay4FuzzRadio, self.mainWindow.wfPay4PayloadBox, self.mainWindow.wfPay4StaticRadio, self.mainWindow.wfPay4DynamicRadio, self.mainWindow.wfPay4StaticEdit)
self.save_config_configuration_item('Payload5', self.mainWindow.wfPay5FuzzRadio, self.mainWindow.wfPay5PayloadBox, self.mainWindow.wfPay5StaticRadio, self.mainWindow.wfPay5DynamicRadio, self.mainWindow.wfPay5StaticEdit)
def save_config_configuration_item(self, payload_item, fuzzRadio, payloadBox, staticRadio, dynamicRadio, staticEdit):
self.framework.set_raft_config_value('WebFuzzer.Config.{0}FuzzSelected'.format(payload_item), fuzzRadio.isChecked())
self.framework.set_raft_config_value('WebFuzzer.Config.{0}FuzzPayload'.format(payload_item), str(payloadBox.currentText()))
self.framework.set_raft_config_value('WebFuzzer.Config.{0}StaticSelected'.format(payload_item), staticRadio.isChecked())
self.framework.set_raft_config_value('WebFuzzer.Config.{0}DynamicSelected'.format(payload_item), dynamicRadio.isChecked())
self.framework.set_raft_config_value('WebFuzzer.Config.{0}StaticEdit'.format(payload_item), staticEdit.text())
def start_fuzzing_clicked(self):
""" Start the fuzzing attack """
if 'Cancel' == self.mainWindow.wfStdStartButton.text() and self.pending_fuzz_requests is not None:
self.cancel_fuzz_requests = True
for context, pending_request in self.pending_fuzz_requests.items():
pending_request.cancel()
self.pending_fuzz_requests = None
self.mainWindow.wfStdStartButton.setText('Start Attack')
self.mainWindow.fuzzerStandardProgressBar.setValue(0)
return
self.pending_fuzz_requests = {}
url = str(self.mainWindow.wfStdUrlEdit.text())
templateText = str(self.mainWindow.wfStdEdit.toPlainText())
method = str(self.mainWindow.stdFuzzerReqMethod.currentText())
self.save_standard_configuration()
replacements = self.build_replacements(method, url)
sequenceId = None
if self.mainWindow.wfStdPreChk.isChecked():
sequenceId = self.mainWindow.wfStdPreBox.itemData(self.mainWindow.wfStdPreBox.currentIndex())
postSequenceId = None
if self.mainWindow.wfStdPostChk.isChecked():
postSequenceId = self.mainWindow.wfStdPostBox.itemData(self.mainWindow.wfStdPostBox.currentIndex())
# Fuzzing stuff
payload_mapping = self.create_payload_map()
# print(payload_mapping)
self.create_functions()
template_definition = TemplateDefinition(templateText)
template_items = template_definition.template_items
### print(template_items)
parameter_names = template_definition.parameter_names
self.global_ns = self.local_ns = {}
scriptLoader = ScriptLoader()
errors = []
fuzz_payloads = {}
for name, payload_info in payload_mapping.items():
if name in parameter_names:
payload_type, payload_value, payload_file = payload_info
if 'fuzz' == payload_type:
filename = payload_value
values = self.Attacks.read_data(filename)
fuzz_payloads[name] = values
elif 'dynamic' == payload_type:
target = payload_file
# TODO: should this come from saved file or current Scintilla values (?)
script_env = scriptLoader.load_from_file(os.path.join(self.functions_dir, target), self.global_ns, self.local_ns)
expression = payload_value
if not expression.endswith('()'):
expression += '()'
eval_result = eval(expression, self.global_ns, self.local_ns)
fuzz_payloads[name] = [str(v) for v in eval_result]
elif 'static' == payload_type:
pass
elif 'none' == payload_type:
# unconfigured payload
errors.append(name)
test_slots = []
counters = []
tests_count = []
total_tests = 1
for name, payload_info in payload_mapping.items():
if name in parameter_names:
payload_type, payload_value, payload_file = payload_info
if 'static' == payload_type:
# static payload value
payloads = [payload_value]
elif 'fuzz' == payload_type:
payloads = fuzz_payloads[name]
elif 'dynamic' == payload_type:
payloads = fuzz_payloads[name]
total_tests *= len(payloads)
test_slots.append((name, payloads))
counters.append(0)
tests_count.append(len(payloads))
position_end = len(counters) - 1
position = position_end
self.miniResponseRenderWidget.clear_response_render()
self.mainWindow.fuzzerStandardProgressBar.setValue(0)
self.mainWindow.fuzzerStandardProgressBar.setMaximum(total_tests)
finished = False
first = True
while not finished:
data = {}
for j in range(0, len(test_slots)):
name, payloads = test_slots[j]
data[name] = payloads[counters[j]]
template_io = StringIO()
self.apply_template_parameters(template_io, data, template_items)
templateText = template_io.getvalue()
context = uuid.uuid4().hex
# print('%s%s%s' % ('-'*32, request, '-'*32))
use_global_cookie_jar = self.mainWindow.webFuzzerUseGlobalCookieJar.isChecked()
(method, url, headers, body) = self.process_template(url, templateText, replacements)
if first:
self.mainWindow.wfStdStartButton.setText('Cancel')
if use_global_cookie_jar:
self.fuzzRequesterCookieJar = self.framework.get_global_cookie_jar()
else:
self.fuzzRequesterCookieJar = InMemoryCookieJar(self.framework, self)
self.requestRunner = RequestRunner(self.framework, self)
self.requestRunner.setup(self.fuzzer_response_received, self.fuzzRequesterCookieJar, sequenceId, postSequenceId)
first = False
self.pending_fuzz_requests[context] = self.requestRunner.queue_request(method, url, headers, body, context)
# increment to next test
counters[position] = (counters[position] + 1) % (tests_count[position])
while position >= 0 and counters[position] == 0:
position -= 1
counters[position] = (counters[position] + 1) % (tests_count[position])
if position == -1:
finished = True
else:
position = position_end
def apply_template_parameters(self, template_io, data, template_items):
for item in template_items:
if item.is_text():
template_io.write(item.item_value)
elif item.is_builtin():
template_io.write('${'+item.item_value+'}')
elif item.is_payload():
template_io.write(data[item.item_value])
elif item.is_function():
temp_io = StringIO()
self.apply_template_parameters(temp_io, data, item.items)
temp_result = temp_io.getvalue()
temp_io = None
result = eval('%s(%s)' % (item.item_value, repr(temp_result)), self.global_ns, self.local_ns)
template_io.write(str(result))
else:
raise Exception('unsupported template parameters: ' + repr(item))
def build_replacements(self, method, url):
replacements = {}
splitted = urlparse.urlsplit(url)
replacements['method'] = method.upper()
replacements['url'] = url
replacements['scheme'] = splitted.scheme or ''
replacements['netloc'] = splitted.netloc or ''
replacements['host'] = splitted.hostname or ''
replacements['path'] = splitted.path or ''
replacements['query'] = splitted.query or ''
replacements['fragment'] = splitted.fragment or ''
replacements['request_uri'] = url
replacements['user_agent'] = self.framework.useragent()
return replacements
def process_template(self, url, template, replacements):
# Start of old
method, uri = '' ,''
headers, body = '', ''
# TODO: this allows for missing entries -- is this good?
func = lambda m: replacements.get(m.group(1))
prev = 0
while True:
n = template.find('\n', prev)
if -1 == n:
break
if n > 0 and '\r' == template[n-1]:
line = template[prev:n-1]
else:
line = template[prev:n]
if 0 == len(line):
# end of headers
headers = template[0:n+1]
body = template[n+1:]
break
prev = n + 1
if not headers:
headers = template
body = ''
# TODO: could work from ordered dict to main order?
headers_dict = {}
first = True
for line in headers.splitlines():
# print(line)
if not line:
break
if '$' in line:
line = self.re_replacement.sub(func, line)
if first:
m = self.re_request.match(line)
if not m:
raise Exception('Invalid HTTP request: failed to match request line: %s' % (line))
method = m.group(1)
uri = m.group(2)
first = False
continue
if ':' in line:
name, value = [v.strip() for v in line.split(':', 1)]
headers_dict[name] = value
if '$' in body:
body = self.re_replacement.sub(func, body)
url = urlparse.urljoin(url, uri)
return (method, url, headers_dict, body)
def fuzzer_history_clear_button_clicked(self):
self.Data.clear_fuzzer_history(self.cursor)
self.fuzzerHistoryDataModel.clearModel()
def fuzzer_response_received(self, response_id, context):
self.mainWindow.fuzzerStandardProgressBar.setValue(self.mainWindow.fuzzerStandardProgressBar.value()+1)
context = str(context)
if self.pending_fuzz_requests is not None:
try:
self.pending_fuzz_requests.pop(context)
except KeyError as e:
pass
if 0 != response_id:
row = self.Data.read_responses_by_id(self.cursor, response_id)
if row:
response_item = [m or '' for m in row]
self.Data.insert_fuzzer_history(self.cursor, response_id)
self.fuzzerHistoryDataModel.append_data([response_item])
finished = False
if self.pending_fuzz_requests is None or len(self.pending_fuzz_requests) == 0:
self.mainWindow.fuzzerStandardProgressBar.setValue(self.mainWindow.fuzzerStandardProgressBar.maximum())
finished = True
elif self.mainWindow.fuzzerStandardProgressBar.value() == self.mainWindow.fuzzerStandardProgressBar.maximum():
finished = True
if finished:
self.mainWindow.wfStdStartButton.setText('Start Attack')
class RequesterTab(QObject):
def __init__(self, framework, mainWindow):
QObject.__init__(self, mainWindow)
self.framework = framework
self.mainWindow = mainWindow
self.mainWindow.requesterSendButton.clicked.connect(self.requester_send_button_clicked)
self.mainWindow.bulkRequestPushButton.clicked.connect(self.requester_bulk_request_button_clicked)
self.mainWindow.requesterHistoryClearButton.clicked.connect(self.requester_history_clear_button_clicked)
self.mainWindow.reqTabWidget.currentChanged.connect(self.handle_tab_currentChanged)
self.mainWindow.requesterSequenceCheckBox.stateChanged.connect(self.handle_requesterSequenceCheckBox_stateChanged)
self.mainWindow.bulkRequestSequenceCheckBox.stateChanged.connect(self.handle_bulkRequestSequenceCheckBox_stateChanged)
self.mainWindow.sequenceRunnerRunButton.clicked.connect(self.handle_sequenceRunnerRunButton_clicked)
self.pending_request = None
self.pending_bulk_requests = None
self.pending_sequence_requests = None
self.re_request = re.compile(r'^(\S+)\s+((?:https?://(?:\S+\.)+\w+(?::\d+)?)?/.*)\s+HTTP/\d+\.\d+\s*$', re.I)
self.re_request_cookie = re.compile(r'^Cookie:\s*(\S+)', re.I|re.M)
self.re_replacement = re.compile(r'\$\{(\w+)\}')
self.framework.subscribe_populate_requester_response_id(self.requester_populate_response_id)
self.framework.subscribe_populate_bulk_requester_responses(self.bulk_requester_populate_responses)
self.framework.subscribe_sequences_changed(self.fill_sequences)
self.setup_requester_tab()
self.Data = None
self.cursor = None
self.framework.subscribe_database_events(self.db_attach, self.db_detach)
def db_attach(self):
self.Data = self.framework.getDB()
self.cursor = self.Data.allocate_thread_cursor()
self.fill_requesters()
def db_detach(self):
self.close_cursor()
self.Data = None
def close_cursor(self):
if self.cursor and self.Data:
self.cursor.close()
self.Data.release_thread_cursor(self.cursor)
self.cursor = None
def setup_requester_tab(self):
self.historyRequestResponse = RequestResponseWidget(self.framework, self.mainWindow.requesterHistoryTabWidget, self.mainWindow.requesterHistorySearchResultsPlaceholder, self)
self.requesterHistoryDataModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
self.mainWindow.requesterHistoryTreeView.setModel(self.requesterHistoryDataModel)
self.mainWindow.requesterHistoryTreeView.activated.connect(self.fill_history_request_response)
self.mainWindow.requesterHistoryTreeView.clicked.connect(self.fill_history_request_response)
self.mainWindow.requesterHistoryTreeView.doubleClicked.connect(self.requester_history_item_double_clicked)
self.historyResponsesContextMenu = ResponsesContextMenuWidget(self.framework, self.requesterHistoryDataModel, self.mainWindow.requesterHistoryTreeView, self)
self.historyResponsesContextMenu.set_currentChanged_callback(self.fill_history_request_response)
self.sequenceRunnerRequestResponse = RequestResponseWidget(self.framework, self.mainWindow.sequenceRunnerTabWidget, self.mainWindow.sequenceRunnerSearchResultsPlaceholder, self)
self.sequenceRunnerDataModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
self.mainWindow.sequenceRunnerTreeView.setModel(self.sequenceRunnerDataModel)
self.mainWindow.sequenceRunnerTreeView.activated.connect(self.fill_sequence_runner_request_response)
self.mainWindow.sequenceRunnerTreeView.clicked.connect(self.fill_sequence_runner_request_response)
self.mainWindow.sequenceRunnerTreeView.doubleClicked.connect(self.requester_sequence_runner_item_double_clicked)
self.sequence_runnerResponsesContextMenu = ResponsesContextMenuWidget(self.framework, self.sequenceRunnerDataModel, self.mainWindow.sequenceRunnerTreeView, self)
self.sequence_runnerResponsesContextMenu.set_currentChanged_callback(self.fill_sequence_runner_request_response)
self.miniResponseRenderWidget = MiniResponseRenderWidget(self.framework, self.mainWindow.reqRespTabWidget, True, self)
self.scopeController = self.framework.getScopeController()
def requester_history_item_double_clicked(self, index):
Id = interface.index_to_id(self.requesterHistoryDataModel, index)
if Id:
dialog = RequestResponseDetailDialog(self.framework, Id, self.mainWindow)
dialog.show()
dialog.exec_()
def fill_history_request_response(self, index):
Id = interface.index_to_id(self.requesterHistoryDataModel, index)
if Id:
self.historyRequestResponse.fill(Id)
def requester_sequence_runner_item_double_clicked(self, index):
Id = interface.index_to_id(self.sequenceRunnerDataModel, index)
if Id:
dialog = RequestResponseDetailDialog(self.framework, Id, self.mainWindow)
dialog.show()
dialog.exec_()
def fill_sequence_runner_request_response(self, index):
Id = interface.index_to_id(self.sequenceRunnerDataModel, index)
if Id:
self.sequenceRunnerRequestResponse.fill(Id)
def fill_requesters(self):
# requesters
self.requesterHistoryDataModel.clearModel()
history_items = []
for row in self.Data.get_all_requester_history(self.cursor):
response_item = interface.data_row_to_response_items(row)
history_items.append(response_item)
self.requesterHistoryDataModel.append_data(history_items)
self.fill_sequences()
self.mainWindow.requesterUrlEdit.setText(self.framework.get_raft_config_value('requesterUrlEdit'))
self.mainWindow.bulkRequestUrlListEdit.setPlainText(self.framework.get_raft_config_value('bulkRequestUrlListEdit'))
def fill_sequences(self):
self.fill_sequences_combo_box(self.mainWindow.requesterSequenceComboBox)
self.fill_sequences_combo_box(self.mainWindow.bulkRequestSequenceComboBox)
self.fill_sequences_combo_box(self.mainWindow.sequenceRunnerSequenceComboBox)
def requester_populate_response_id(self, Id):
row = self.Data.read_responses_by_id(self.cursor, Id)
if not row:
return
responseItems = interface.data_row_to_response_items(row)
method, url, template_text = self.generate_template_for_response_item(responseItems)
self.set_combo_box_text(self.mainWindow.requesterRequestMethod, method.upper())
self.mainWindow.requesterUrlEdit.setText(url)
self.mainWindow.requesterTemplateEdit.setPlainText(template_text)
def bulk_requester_populate_responses(self, id_list):
url_list = []
first = True
for Id in id_list:
row = self.Data.read_responses_by_id(self.cursor, Id)
if not row:
continue
responseItems = interface.data_row_to_response_items(row)
url = responseItems[ResponsesTable.URL]
if url not in url_list:
url_list.append(url)
if first:
method, url, template_text = self.generate_template_for_response_item(responseItems)
self.set_combo_box_text(self.mainWindow.bulkRequestMethodEdit, method.upper())
self.mainWindow.bulkRequestTemplateEdit.setPlainText(template_text)
first = False
self.mainWindow.bulkRequestUrlListEdit.setPlainText('\n'.join(url_list))
def generate_template_for_response_item(self, responseItems):
url = responseItems[ResponsesTable.URL]
reqHeaders = str(responseItems[ResponsesTable.REQ_HEADERS], 'utf-8', 'ignore')
reqData = str(responseItems[ResponsesTable.REQ_DATA], 'utf-8', 'ignore')
method = responseItems[ResponsesTable.REQ_METHOD]
splitted = urlparse.urlsplit(url)
useragent = self.framework.useragent()
has_cookie = False
template = StringIO()
template.write('${method} ${request_uri} HTTP/1.1\n')
first = True
for line in reqHeaders.splitlines():
if not line:
break
if first and self.re_request.match(line):
first = False
continue
if ':' in line:
name, value = [v.strip() for v in line.split(':', 1)]
lname = name.lower()
if 'host' == lname:
if splitted.hostname and value == splitted.hostname:
template.write('Host: ${host}\n')
continue
elif 'user-agent' == lname:
if useragent == value:
template.write('User-Agent: ${user_agent}\n')
continue
template.write(line)
template.write('\n')
template.write('\n')
template.write(reqData)
return method, url, template.getvalue()
def set_combo_box_text(self, comboBox, selectedText):
index = comboBox.findText(selectedText)
if -1 == index:
comboBox.addItem(selectedText)
index = comboBox.findText(selectedText)
comboBox.setCurrentIndex(index)
def handle_requesterSequenceCheckBox_stateChanged(self, state):
self.mainWindow.requesterSequenceComboBox.setEnabled(self.mainWindow.requesterSequenceCheckBox.isChecked())
def handle_bulkRequestSequenceCheckBox_stateChanged(self, state):
self.mainWindow.bulkRequestSequenceComboBox.setEnabled(self.mainWindow.bulkRequestSequenceCheckBox.isChecked())
def handle_tab_currentChanged(self, index):
# TODO: must this hard-coded ?
if 0 == index:
self.fill_sequences_combo_box(self.mainWindow.requesterSequenceComboBox)
elif 1 == index:
self.fill_sequences_combo_box(self.mainWindow.bulkRequestSequenceComboBox)
elif 2 == index:
self.fill_sequences_combo_box(self.mainWindow.sequenceRunnerSequenceComboBox)
def requester_send_button_clicked(self):
""" Make a request from the Request tab """
if 'Cancel' == self.mainWindow.requesterSendButton.text() and self.pending_request is not None:
self.pending_request.cancel()
self.pending_request = None
self.mainWindow.requesterSendButton.setText('Send')
return
qurl = QUrl.fromUserInput(self.mainWindow.requesterUrlEdit.text())
url = qurl.toEncoded().data().decode('utf-8')
self.mainWindow.requesterUrlEdit.setText(url)
self.framework.set_raft_config_value('requesterUrlEdit', url)
templateText = str(self.mainWindow.requesterTemplateEdit.toPlainText())
method = str(self.mainWindow.requesterRequestMethod.currentText())
use_global_cookie_jar = self.mainWindow.requesterUseGlobalCookieJar.isChecked()
replacements = self.build_replacements(method, url)
(method, url, headers, body) = self.process_template(url, templateText, replacements)
sequenceId = None
if self.mainWindow.requesterSequenceCheckBox.isChecked():
sequenceId = str(self.mainWindow.requesterSequenceComboBox.itemData(self.mainWindow.requesterSequenceComboBox.currentIndex()))
self.requestRunner = RequestRunner(self.framework, self)
if use_global_cookie_jar:
self.requesterCookieJar = self.framework.get_global_cookie_jar()
else:
self.requesterCookieJar = InMemoryCookieJar(self.framework, self)
self.requestRunner.setup(self.requester_response_received, self.requesterCookieJar, sequenceId)
self.pending_request = self.requestRunner.queue_request(method, url, headers, body)
self.mainWindow.requesterSendButton.setText('Cancel')
self.miniResponseRenderWidget.clear_response_render()
def requester_response_received(self, response_id, context):
if 0 != response_id:
row = self.Data.read_responses_by_id(self.cursor, response_id)
if row:
response_item = interface.data_row_to_response_items(row)
self.Data.insert_requester_history(self.cursor, response_id)
self.requesterHistoryDataModel.append_data([response_item])
url = response_item[ResponsesTable.URL]
req_headers = response_item[ResponsesTable.REQ_HEADERS]
req_body = response_item[ResponsesTable.REQ_DATA]
res_headers = response_item[ResponsesTable.RES_HEADERS]
res_body = response_item[ResponsesTable.RES_DATA]
res_content_type = response_item[ResponsesTable.RES_CONTENT_TYPE]
self.miniResponseRenderWidget.populate_response_content(url, req_headers, req_body, res_headers, res_body, res_content_type)
self.mainWindow.requesterSendButton.setText('Send')
self.pending_request = None
def requester_bulk_request_button_clicked(self):
if 'Cancel' == self.mainWindow.bulkRequestPushButton.text() and self.pending_bulk_requests is not None:
self.cancel_bulk_requests = True
for context, pending_request in self.pending_bulk_requests.items():
pending_request.cancel()
self.pending_bulk_requests = None
self.mainWindow.bulkRequestPushButton.setText('Send')
self.mainWindow.bulkRequestProgressBar.setValue(0)
return
if self.pending_bulk_requests is None:
self.pending_bulk_requests = {}
method = str(self.mainWindow.bulkRequestMethodEdit.currentText())
templateText = str(self.mainWindow.bulkRequestTemplateEdit.toPlainText())
template_url = str(self.mainWindow.bulkRequestUrlEdit.text())
url_list = str(self.mainWindow.bulkRequestUrlListEdit.toPlainText())
self.framework.set_raft_config_value('bulkRequestUrlListEdit', url_list)
request_urls = url_list.splitlines()
self.mainWindow.bulkRequestProgressBar.setValue(0)
self.mainWindow.bulkRequestProgressBar.setMaximum(len(request_urls))
sequenceId = None
if self.mainWindow.bulkRequestSequenceCheckBox.isChecked():
sequenceId = str(self.mainWindow.bulkRequestSequenceComboBox.itemData(self.mainWindow.bulkRequestSequenceComboBox.currentIndex()))
first = True
self.cancel_bulk_requests = False
for request_url in request_urls:
if self.cancel_bulk_requests:
break
request_url = request_url.strip()
if request_url:
context = uuid.uuid4().hex
# TODO: move this hack
if '$' in template_url:
replacements = self.build_replacements(method, request_url)
url = self.re_replacement.sub(lambda m: replacements.get(m.group(1)), template_url)
else:
url = request_url
if not self.scopeController.isUrlInScope(url, url):
self.framework.log_warning('skipping out of scope URL: %s' % (url))
self.mainWindow.bulkRequestProgressBar.setValue(self.mainWindow.bulkRequestProgressBar.value()+1)
continue
use_global_cookie_jar = self.mainWindow.bulkRequestUseGlobalCookieJar.isChecked()
replacements = self.build_replacements(method, url)
(method, url, headers, body) = self.process_template(url, templateText, replacements)
if first:
self.mainWindow.bulkRequestPushButton.setText('Cancel')
if use_global_cookie_jar:
self.bulkRequesterCookieJar = self.framework.get_global_cookie_jar()
else:
self.bulkRequesterCookieJar = InMemoryCookieJar(self.framework, self)
self.bulk_requestRunner = RequestRunner(self.framework, self)
self.bulk_requestRunner.setup(self.requester_bulk_response_received, self.bulkRequesterCookieJar, sequenceId)
first = False
self.pending_bulk_requests[context] = self.bulk_requestRunner.queue_request(method, url, headers, body, context)
def requester_bulk_response_received(self, response_id, context):
self.mainWindow.bulkRequestProgressBar.setValue(self.mainWindow.bulkRequestProgressBar.value()+1)
context = str(context)
if self.pending_bulk_requests is not None:
try:
self.pending_bulk_requests.pop(context)
except KeyError as e:
pass
if 0 != response_id:
row = self.Data.read_responses_by_id(self.cursor, response_id)
if row:
response_item = interface.data_row_to_response_items(row)
self.Data.insert_requester_history(self.cursor, response_id)
self.requesterHistoryDataModel.append_data([response_item])
finished = False
if self.pending_bulk_requests is None or len(self.pending_bulk_requests) == 0:
self.mainWindow.bulkRequestProgressBar.setValue(self.mainWindow.bulkRequestProgressBar.maximum())
finished = True
elif self.mainWindow.bulkRequestProgressBar.value() == self.mainWindow.bulkRequestProgressBar.maximum():
finished = True
if finished:
self.mainWindow.bulkRequestPushButton.setText('Send')
def handle_sequenceRunnerRunButton_clicked(self):
""" Run a sequence """
if 'Cancel' == self.mainWindow.sequenceRunnerRunButton.text() and self.pending_sequence_requests is not None:
self.cancel_sequence_requests = True
for context, pending_request in self.pending_sequence_requests.items():
pending_request.cancel()
self.pending_sequence_requests = None
self.mainWindow.sequenceRunnerButton.setText('Send')
self.mainWindow.sequenceRunnerButton.setValue(0)
return
self.sequenceRunnerDataModel.clearModel()
sequenceId = str(self.mainWindow.sequenceRunnerSequenceComboBox.itemData(self.mainWindow.sequenceRunnerSequenceComboBox.currentIndex()))
use_global_cookie_jar = self.mainWindow.sequenceRunnerUseGlobalCookieJar.isChecked()
if use_global_cookie_jar:
self.sequenceRunnerCookieJar = self.framework.get_global_cookie_jar()
else:
self.sequenceRunnerCookieJar = InMemoryCookieJar(self.framework, self)
self.sequence_requestRunner = RequestRunner(self.framework, self)
self.sequence_requestRunner.setup(self.sequence_runner_response_received, self.sequenceRunnerCookieJar, sequenceId)
self.pending_sequence_requests = self.sequence_requestRunner.run_sequence()
self.mainWindow.sequenceRunnerRunButton.setText('Cancel')
def sequence_runner_response_received(self, response_id, context):
context = str(context)
if self.pending_sequence_requests is not None:
try:
self.pending_sequence_requests.pop(context)
except KeyError as e:
print((e))
pass
if 0 != response_id:
row = self.Data.read_responses_by_id(self.cursor, response_id)
if row:
response_item = interface.data_row_to_response_items(row)
self.sequenceRunnerDataModel.append_data([response_item])
if self.pending_sequence_requests is None or len(self.pending_sequence_requests) == 0:
self.mainWindow.sequenceRunnerRunButton.setText('Send')
def requester_history_clear_button_clicked(self):
self.Data.clear_requester_history(self.cursor)
self.requesterHistoryDataModel.clearModel()
def fill_sequences_combo_box(self, comboBox):
selectedText = comboBox.currentText()
comboBox.clear()
for row in self.Data.get_all_sequences(self.cursor):
sequenceItem = [m or '' for m in row]
name = str(sequenceItem[1])
Id = str(sequenceItem[0])
item = comboBox.addItem(name, Id)
if selectedText:
index = comboBox.findText(selectedText)
if index != -1:
comboBox.setCurrentIndex(index)
def build_replacements(self, method, url):
replacements = {}
splitted = urlparse.urlsplit(url)
replacements['method'] = method.upper()
replacements['url'] = url
replacements['scheme'] = splitted.scheme or ''
replacements['netloc'] = splitted.netloc or ''
replacements['host'] = splitted.hostname or ''
replacements['path'] = splitted.path or '/'
replacements['query'] = splitted.query or ''
replacements['fragment'] = splitted.fragment or ''
replacements['request_uri'] = urlparse.urlunsplit(('', '', replacements['path'], replacements['query'], ''))
replacements['user_agent'] = self.framework.useragent()
return replacements
def process_template(self, url, template, replacements):
method, uri = '' ,''
headers, body = '', ''
# TODO: this allows for missing entries -- is this good?
func = lambda m: replacements.get(m.group(1))
prev = 0
while True:
n = template.find('\n', prev)
if -1 == n:
break
if n > 0 and '\r' == template[n-1]:
line = template[prev:n-1]
else:
line = template[prev:n]
if 0 == len(line):
# end of headers
headers = template[0:n+1]
body = template[n+1:]
break
prev = n + 1
if not headers:
headers = template
body = ''
# TODO: could work from ordered dict to main order?
headers_dict = {}
first = True
for line in headers.splitlines():
if not line:
break
if '$' in line:
line = self.re_replacement.sub(func, line)
if first:
m = self.re_request.match(line)
if not m:
raise Exception('Invalid HTTP request: failed to match request line: %s' % (line))
method = m.group(1)
uri = m.group(2)
first = False
continue
if ':' in line:
name, value = [v.strip() for v in line.split(':', 1)]
headers_dict[name] = value
if '$' in body:
body = self.re_replacement.sub(func, body)
url = urlparse.urljoin(url, uri)
return (method, url, headers_dict, body)
def requester_bulk_request_button_clicked(self):
if 'Cancel' == self.mainWindow.bulkRequestPushButton.text() and self.pending_bulk_requests is not None:
self.cancel_bulk_requests = True
for context, pending_request in self.pending_bulk_requests.items():
pending_request.cancel()
self.pending_bulk_requests = None
self.mainWindow.bulkRequestPushButton.setText('Send')
self.mainWindow.bulkRequestProgressBar.setValue(0)
return
if self.pending_bulk_requests is None:
self.pending_bulk_requests = {}
method = str(self.mainWindow.bulkRequestMethodEdit.currentText())
templateText = str(self.mainWindow.bulkRequestTemplateEdit.toPlainText())
template_url = str(self.mainWindow.bulkRequestUrlEdit.text())
url_list = str(self.mainWindow.bulkRequestUrlListEdit.toPlainText())
self.framework.set_raft_config_value('bulkRequestUrlListEdit', url_list)
request_urls = url_list.splitlines()
self.mainWindow.bulkRequestProgressBar.setValue(0)
self.mainWindow.bulkRequestProgressBar.setMaximum(len(request_urls))
sequenceId = None
if self.mainWindow.bulkRequestSequenceCheckBox.isChecked():
sequenceId = str(self.mainWindow.bulkRequestSequenceComboBox.itemData(self.mainWindow.bulkRequestSequenceComboBox.currentIndex()))
first = True
self.cancel_bulk_requests = False
for request_url in request_urls:
if self.cancel_bulk_requests:
break
request_url = request_url.strip()
if request_url:
context = uuid.uuid4().hex
# TODO: move this hack
if '$' in template_url:
replacements = self.build_replacements(method, request_url)
url = self.re_replacement.sub(lambda m: replacements.get(m.group(1)), template_url)
else:
url = request_url
if not self.scopeController.isUrlInScope(url, url):
self.framework.log_warning('skipping out of scope URL: %s' % (url))
self.mainWindow.bulkRequestProgressBar.setValue(self.mainWindow.bulkRequestProgressBar.value()+1)
continue
use_global_cookie_jar = self.mainWindow.bulkRequestUseGlobalCookieJar.isChecked()
replacements = self.build_replacements(method, url)
(method, url, headers, body) = self.process_template(url, templateText, replacements)
if first:
self.mainWindow.bulkRequestPushButton.setText('Cancel')
if use_global_cookie_jar:
self.bulkRequesterCookieJar = self.framework.get_global_cookie_jar()
else:
self.bulkRequesterCookieJar = InMemoryCookieJar(self.framework, self)
self.bulk_requestRunner = RequestRunner(self.framework, self)
self.bulk_requestRunner.setup(self.requester_bulk_response_received, self.bulkRequesterCookieJar, sequenceId)
first = False
self.pending_bulk_requests[context] = self.bulk_requestRunner.queue_request(method, url, headers, body, context)
