示例#1
0
    def __init__(self, framework, parent=None):
        super(SearchDialog, self).__init__(parent)
        self.setupUi(self)

        self.connect(self, SIGNAL('finished()'), self.finishedHandler)

        self.framework = framework

        # progress dialog
        self.Progress = ProgressDialog(self)
        QObject.connect(self.Progress, SIGNAL('canceled()'), self.canceledHandler)

        self.searchRequestResponse = RequestResponseWidget(self.framework, self.searchTabWidget, self.searchSearchControlPlaceholder, self)

        self.searchResultsModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
        self.searchResultsTree.setModel(self.searchResultsModel)
        self.searchResultsTree.clicked.connect(self.fill_bottom)
        self.searchResultsTree.doubleClicked.connect(self.response_item_double_clicked)
        self.searchResultsTree.setSortingEnabled(True)
        self.searchResultsTree.sortByColumn(0, Qt.AscendingOrder)

        self.thread = SearchThread(self.framework, self.searchResultsModel)
        self.thread.start(QThread.LowestPriority)
        self.finished.connect(self.finishedHandler)
        self.searchPushButton.pressed.connect(self.startSearch)
        self.connect(self, SIGNAL('searchFinished()'), self.searchFinishedHandler, Qt.QueuedConnection)
        # Create context menu
        self.resultsContextMenu = ResponsesContextMenuWidget(self.framework, self.searchResultsModel, self.searchResultsTree, self)
        self.resultsContextMenu.set_currentChanged_callback(self.fill_bottom)
示例#2
0
    def setup_requester_tab(self):

        self.historyRequestResponse = RequestResponseWidget(
            self.framework, self.mainWindow.requesterHistoryTabWidget,
            self.mainWindow.requesterHistorySearchResultsPlaceholder, self)
        self.requesterHistoryDataModel = ResponsesDataModel.ResponsesDataModel(
            self.framework, self)
        self.mainWindow.requesterHistoryTreeView.setModel(
            self.requesterHistoryDataModel)
        self.mainWindow.requesterHistoryTreeView.activated.connect(
            self.fill_history_request_response)
        self.mainWindow.requesterHistoryTreeView.clicked.connect(
            self.fill_history_request_response)
        self.mainWindow.requesterHistoryTreeView.doubleClicked.connect(
            self.requester_history_item_double_clicked)
        self.historyResponsesContextMenu = ResponsesContextMenuWidget(
            self.framework, self.requesterHistoryDataModel,
            self.mainWindow.requesterHistoryTreeView, self)
        self.historyResponsesContextMenu.set_currentChanged_callback(
            self.fill_history_request_response)

        self.sequenceRunnerRequestResponse = RequestResponseWidget(
            self.framework, self.mainWindow.sequenceRunnerTabWidget,
            self.mainWindow.sequenceRunnerSearchResultsPlaceholder, self)
        self.sequenceRunnerDataModel = ResponsesDataModel.ResponsesDataModel(
            self.framework, self)
        self.mainWindow.sequenceRunnerTreeView.setModel(
            self.sequenceRunnerDataModel)
        self.mainWindow.sequenceRunnerTreeView.activated.connect(
            self.fill_sequence_runner_request_response)
        self.mainWindow.sequenceRunnerTreeView.clicked.connect(
            self.fill_sequence_runner_request_response)
        self.mainWindow.sequenceRunnerTreeView.doubleClicked.connect(
            self.requester_sequence_runner_item_double_clicked)
        self.sequence_runnerResponsesContextMenu = ResponsesContextMenuWidget(
            self.framework, self.sequenceRunnerDataModel,
            self.mainWindow.sequenceRunnerTreeView, self)
        self.sequence_runnerResponsesContextMenu.set_currentChanged_callback(
            self.fill_sequence_runner_request_response)

        self.miniResponseRenderWidget = MiniResponseRenderWidget(
            self.framework, self.mainWindow.reqRespTabWidget, True, self)

        self.scopeController = self.framework.getScopeController()
    def __init__(self, framework, responseId, parent = None):
        super(RequestResponseDetailDialog, self).__init__(parent)
        self.setupUi(self)
        self.framework = framework
        self.responseId = responseId

        dialogTitle = "Response Detail - #%s" % (responseId)
        self.setWindowTitle(QApplication.translate("RequestResponseDetailDialog", dialogTitle, None, QApplication.UnicodeUTF8))

        self.requestResponseWidgetDetail = RequestResponseDetailWidget(self.framework, self.detailWidget, responseId, self)
        # TODO: consider if search and update widget is needed
        self.requestResponseWidget = RequestResponseWidget(self.framework, self.tabWidget, None, self)
        self.requestResponseWidget.fill(responseId)

        if parent:
            frameGeometry = parent.frameGeometry()
            self.setMaximumWidth(frameGeometry.width() - 40)
            self.setMaximumHeight(frameGeometry.height() - 20)
            self.detailWidget.setMaximumWidth(frameGeometry.width() - 40)
            self.detailWidget.setMaximumHeight(frameGeometry.height() - 20)
class RequestResponseDetailDialog(QDialog, RequestResponseDetailDialog.Ui_RequestResponseDetailDialog):
    def __init__(self, framework, responseId, parent = None):
        super(RequestResponseDetailDialog, self).__init__(parent)
        self.setupUi(self)
        self.framework = framework
        self.responseId = responseId

        dialogTitle = "Response Detail - #%s" % (responseId)
        self.setWindowTitle(QApplication.translate("RequestResponseDetailDialog", dialogTitle, None, QApplication.UnicodeUTF8))

        self.requestResponseWidgetDetail = RequestResponseDetailWidget(self.framework, self.detailWidget, responseId, self)
        # TODO: consider if search and update widget is needed
        self.requestResponseWidget = RequestResponseWidget(self.framework, self.tabWidget, None, self)
        self.requestResponseWidget.fill(responseId)

        if parent:
            frameGeometry = parent.frameGeometry()
            self.setMaximumWidth(frameGeometry.width() - 40)
            self.setMaximumHeight(frameGeometry.height() - 20)
            self.detailWidget.setMaximumWidth(frameGeometry.width() - 40)
            self.detailWidget.setMaximumHeight(frameGeometry.height() - 20)
示例#5
0
    def setup_requester_tab(self):

        self.historyRequestResponse = RequestResponseWidget(self.framework, self.mainWindow.requesterHistoryTabWidget, self.mainWindow.requesterHistorySearchResultsPlaceholder, self)
        self.requesterHistoryDataModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
        self.mainWindow.requesterHistoryTreeView.setModel(self.requesterHistoryDataModel)
        self.mainWindow.requesterHistoryTreeView.activated.connect(self.fill_history_request_response)
        self.mainWindow.requesterHistoryTreeView.clicked.connect(self.fill_history_request_response)
        self.mainWindow.requesterHistoryTreeView.doubleClicked.connect(self.requester_history_item_double_clicked)
        self.historyResponsesContextMenu = ResponsesContextMenuWidget(self.framework, self.requesterHistoryDataModel, self.mainWindow.requesterHistoryTreeView, self)
        self.historyResponsesContextMenu.set_currentChanged_callback(self.fill_history_request_response)

        self.sequenceRunnerRequestResponse = RequestResponseWidget(self.framework, self.mainWindow.sequenceRunnerTabWidget, self.mainWindow.sequenceRunnerSearchResultsPlaceholder, self)
        self.sequenceRunnerDataModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
        self.mainWindow.sequenceRunnerTreeView.setModel(self.sequenceRunnerDataModel)
        self.mainWindow.sequenceRunnerTreeView.activated.connect(self.fill_sequence_runner_request_response)
        self.mainWindow.sequenceRunnerTreeView.clicked.connect(self.fill_sequence_runner_request_response)
        self.mainWindow.sequenceRunnerTreeView.doubleClicked.connect(self.requester_sequence_runner_item_double_clicked)
        self.sequence_runnerResponsesContextMenu = ResponsesContextMenuWidget(self.framework, self.sequenceRunnerDataModel, self.mainWindow.sequenceRunnerTreeView, self)
        self.sequence_runnerResponsesContextMenu.set_currentChanged_callback(self.fill_sequence_runner_request_response)

        self.miniResponseRenderWidget = MiniResponseRenderWidget(self.framework, self.mainWindow.reqRespTabWidget, True, self)

        self.scopeController = self.framework.getScopeController()
示例#6
0
    def __init__(self, framework, parent=None):
        super(SearchDialog, self).__init__(parent)
        self.setupUi(self)

        self.connect(self, SIGNAL('finished()'), self.finishedHandler)

        self.framework = framework

        # progress dialog
        self.Progress = ProgressDialog(self)
        QObject.connect(self.Progress, SIGNAL('canceled()'),
                        self.canceledHandler)

        self.searchRequestResponse = RequestResponseWidget(
            self.framework, self.searchTabWidget,
            self.searchSearchControlPlaceholder, self)

        self.searchResultsModel = ResponsesDataModel.ResponsesDataModel(
            self.framework, self)
        self.searchResultsTree.setModel(self.searchResultsModel)
        self.searchResultsTree.clicked.connect(self.fill_bottom)
        self.searchResultsTree.doubleClicked.connect(
            self.response_item_double_clicked)
        self.searchResultsTree.setSortingEnabled(True)
        self.searchResultsTree.sortByColumn(0, Qt.AscendingOrder)

        self.thread = SearchThread(self.framework, self.searchResultsModel)
        self.thread.start(QThread.LowestPriority)
        self.finished.connect(self.finishedHandler)
        self.searchPushButton.pressed.connect(self.startSearch)
        self.connect(self, SIGNAL('searchFinished()'),
                     self.searchFinishedHandler, Qt.QueuedConnection)
        # Create context menu
        self.resultsContextMenu = ResponsesContextMenuWidget(
            self.framework, self.searchResultsModel, self.searchResultsTree,
            self)
        self.resultsContextMenu.set_currentChanged_callback(self.fill_bottom)
示例#7
0
class RequesterTab(QObject):
    def __init__(self, framework, mainWindow):
        QObject.__init__(self, mainWindow)
        self.framework = framework
        self.mainWindow = mainWindow

        self.mainWindow.requesterSendButton.clicked.connect(
            self.requester_send_button_clicked)
        self.mainWindow.bulkRequestPushButton.clicked.connect(
            self.requester_bulk_request_button_clicked)
        self.mainWindow.requesterHistoryClearButton.clicked.connect(
            self.requester_history_clear_button_clicked)
        self.mainWindow.reqTabWidget.currentChanged.connect(
            self.handle_tab_currentChanged)
        self.mainWindow.requesterSequenceCheckBox.stateChanged.connect(
            self.handle_requesterSequenceCheckBox_stateChanged)
        self.mainWindow.bulkRequestSequenceCheckBox.stateChanged.connect(
            self.handle_bulkRequestSequenceCheckBox_stateChanged)
        self.mainWindow.sequenceRunnerRunButton.clicked.connect(
            self.handle_sequenceRunnerRunButton_clicked)
        self.pending_request = None
        self.pending_bulk_requests = None
        self.pending_sequence_requests = None

        self.re_request = re.compile(
            r'^(\S+)\s+((?:https?://(?:\S+\.)+\w+(?::\d+)?)?/.*)\s+HTTP/\d+\.\d+\s*$',
            re.I)
        self.re_request_cookie = re.compile(r'^Cookie:\s*(\S+)', re.I | re.M)
        self.re_replacement = re.compile(r'\$\{(\w+)\}')

        self.framework.subscribe_populate_requester_response_id(
            self.requester_populate_response_id)
        self.framework.subscribe_populate_bulk_requester_responses(
            self.bulk_requester_populate_responses)
        self.framework.subscribe_sequences_changed(self.fill_sequences)

        self.setup_requester_tab()

        self.Data = None
        self.cursor = None
        self.framework.subscribe_database_events(self.db_attach,
                                                 self.db_detach)

    def db_attach(self):
        self.Data = self.framework.getDB()
        self.cursor = self.Data.allocate_thread_cursor()
        self.fill_requesters()

    def db_detach(self):
        self.close_cursor()
        self.Data = None

    def close_cursor(self):
        if self.cursor and self.Data:
            self.cursor.close()
            self.Data.release_thread_cursor(self.cursor)
            self.cursor = None

    def setup_requester_tab(self):

        self.historyRequestResponse = RequestResponseWidget(
            self.framework, self.mainWindow.requesterHistoryTabWidget,
            self.mainWindow.requesterHistorySearchResultsPlaceholder, self)
        self.requesterHistoryDataModel = ResponsesDataModel.ResponsesDataModel(
            self.framework, self)
        self.mainWindow.requesterHistoryTreeView.setModel(
            self.requesterHistoryDataModel)
        self.mainWindow.requesterHistoryTreeView.activated.connect(
            self.fill_history_request_response)
        self.mainWindow.requesterHistoryTreeView.clicked.connect(
            self.fill_history_request_response)
        self.mainWindow.requesterHistoryTreeView.doubleClicked.connect(
            self.requester_history_item_double_clicked)
        self.historyResponsesContextMenu = ResponsesContextMenuWidget(
            self.framework, self.requesterHistoryDataModel,
            self.mainWindow.requesterHistoryTreeView, self)
        self.historyResponsesContextMenu.set_currentChanged_callback(
            self.fill_history_request_response)

        self.sequenceRunnerRequestResponse = RequestResponseWidget(
            self.framework, self.mainWindow.sequenceRunnerTabWidget,
            self.mainWindow.sequenceRunnerSearchResultsPlaceholder, self)
        self.sequenceRunnerDataModel = ResponsesDataModel.ResponsesDataModel(
            self.framework, self)
        self.mainWindow.sequenceRunnerTreeView.setModel(
            self.sequenceRunnerDataModel)
        self.mainWindow.sequenceRunnerTreeView.activated.connect(
            self.fill_sequence_runner_request_response)
        self.mainWindow.sequenceRunnerTreeView.clicked.connect(
            self.fill_sequence_runner_request_response)
        self.mainWindow.sequenceRunnerTreeView.doubleClicked.connect(
            self.requester_sequence_runner_item_double_clicked)
        self.sequence_runnerResponsesContextMenu = ResponsesContextMenuWidget(
            self.framework, self.sequenceRunnerDataModel,
            self.mainWindow.sequenceRunnerTreeView, self)
        self.sequence_runnerResponsesContextMenu.set_currentChanged_callback(
            self.fill_sequence_runner_request_response)

        self.miniResponseRenderWidget = MiniResponseRenderWidget(
            self.framework, self.mainWindow.reqRespTabWidget, True, self)

        self.scopeController = self.framework.getScopeController()

    def requester_history_item_double_clicked(self, index):
        Id = interface.index_to_id(self.requesterHistoryDataModel, index)
        if Id:
            dialog = RequestResponseDetailDialog(self.framework, Id,
                                                 self.mainWindow)
            dialog.show()
            dialog.exec_()

    def fill_history_request_response(self, index):
        Id = interface.index_to_id(self.requesterHistoryDataModel, index)
        if Id:
            self.historyRequestResponse.fill(Id)

    def requester_sequence_runner_item_double_clicked(self, index):
        Id = interface.index_to_id(self.sequenceRunnerDataModel, index)
        if Id:
            dialog = RequestResponseDetailDialog(self.framework, Id,
                                                 self.mainWindow)
            dialog.show()
            dialog.exec_()

    def fill_sequence_runner_request_response(self, index):
        Id = interface.index_to_id(self.sequenceRunnerDataModel, index)
        if Id:
            self.sequenceRunnerRequestResponse.fill(Id)

    def fill_requesters(self):
        # requesters
        self.requesterHistoryDataModel.clearModel()
        history_items = []
        for row in self.Data.get_all_requester_history(self.cursor):
            response_item = interface.data_row_to_response_items(row)
            history_items.append(response_item)
        self.requesterHistoryDataModel.append_data(history_items)

        self.fill_sequences()

        self.mainWindow.requesterUrlEdit.setText(
            self.framework.get_raft_config_value('requesterUrlEdit'))
        self.mainWindow.bulkRequestUrlListEdit.setPlainText(
            self.framework.get_raft_config_value('bulkRequestUrlListEdit'))

    def fill_sequences(self):
        self.fill_sequences_combo_box(
            self.mainWindow.requesterSequenceComboBox)
        self.fill_sequences_combo_box(
            self.mainWindow.bulkRequestSequenceComboBox)
        self.fill_sequences_combo_box(
            self.mainWindow.sequenceRunnerSequenceComboBox)

    def requester_populate_response_id(self, Id):
        row = self.Data.read_responses_by_id(self.cursor, Id)
        if not row:
            return

        responseItems = interface.data_row_to_response_items(row)

        method, url, template_text = self.generate_template_for_response_item(
            responseItems)

        self.set_combo_box_text(self.mainWindow.requesterRequestMethod,
                                method.upper())
        self.mainWindow.requesterUrlEdit.setText(url)
        self.mainWindow.requesterTemplateEdit.setPlainText(template_text)

    def bulk_requester_populate_responses(self, id_list):

        url_list = []
        first = True
        for Id in id_list:
            row = self.Data.read_responses_by_id(self.cursor, Id)
            if not row:
                continue

            responseItems = interface.data_row_to_response_items(row)
            url = responseItems[ResponsesTable.URL]
            if url not in url_list:
                url_list.append(url)

            if first:
                method, url, template_text = self.generate_template_for_response_item(
                    responseItems)
                self.set_combo_box_text(self.mainWindow.bulkRequestMethodEdit,
                                        method.upper())
                self.mainWindow.bulkRequestTemplateEdit.setPlainText(
                    template_text)
                first = False

        self.mainWindow.bulkRequestUrlListEdit.setPlainText(
            '\n'.join(url_list))

    def generate_template_for_response_item(self, responseItems):
        url = responseItems[ResponsesTable.URL]
        reqHeaders = str(responseItems[ResponsesTable.REQ_HEADERS], 'utf-8',
                         'ignore')
        reqData = str(responseItems[ResponsesTable.REQ_DATA], 'utf-8',
                      'ignore')
        method = responseItems[ResponsesTable.REQ_METHOD]
        splitted = urlparse.urlsplit(url)

        useragent = self.framework.useragent()
        has_cookie = False
        template = StringIO()
        template.write('${method} ${request_uri} HTTP/1.1\n')
        first = True
        for line in reqHeaders.splitlines():
            if not line:
                break
            if first and self.re_request.match(line):
                first = False
                continue
            if ':' in line:
                name, value = [v.strip() for v in line.split(':', 1)]
                lname = name.lower()
                if 'host' == lname:
                    if splitted.hostname and value == splitted.hostname:
                        template.write('Host: ${host}\n')
                        continue
                elif 'user-agent' == lname:
                    if useragent == value:
                        template.write('User-Agent: ${user_agent}\n')
                        continue
            template.write(line)
            template.write('\n')
        template.write('\n')
        template.write(reqData)

        return method, url, template.getvalue()

    def set_combo_box_text(self, comboBox, selectedText):
        index = comboBox.findText(selectedText)
        if -1 == index:
            comboBox.addItem(selectedText)
            index = comboBox.findText(selectedText)
        comboBox.setCurrentIndex(index)

    def handle_requesterSequenceCheckBox_stateChanged(self, state):
        self.mainWindow.requesterSequenceComboBox.setEnabled(
            self.mainWindow.requesterSequenceCheckBox.isChecked())

    def handle_bulkRequestSequenceCheckBox_stateChanged(self, state):
        self.mainWindow.bulkRequestSequenceComboBox.setEnabled(
            self.mainWindow.bulkRequestSequenceCheckBox.isChecked())

    def handle_tab_currentChanged(self, index):
        # TODO: must this hard-coded ?
        if 0 == index:
            self.fill_sequences_combo_box(
                self.mainWindow.requesterSequenceComboBox)
        elif 1 == index:
            self.fill_sequences_combo_box(
                self.mainWindow.bulkRequestSequenceComboBox)
        elif 2 == index:
            self.fill_sequences_combo_box(
                self.mainWindow.sequenceRunnerSequenceComboBox)

    def requester_send_button_clicked(self):
        """ Make a request from the Request tab """

        if 'Cancel' == self.mainWindow.requesterSendButton.text(
        ) and self.pending_request is not None:
            self.pending_request.cancel()
            self.pending_request = None
            self.mainWindow.requesterSendButton.setText('Send')
            return

        qurl = QUrl.fromUserInput(self.mainWindow.requesterUrlEdit.text())
        url = qurl.toEncoded().data().decode('utf-8')
        self.mainWindow.requesterUrlEdit.setText(url)

        self.framework.set_raft_config_value('requesterUrlEdit', url)
        templateText = str(self.mainWindow.requesterTemplateEdit.toPlainText())
        method = str(self.mainWindow.requesterRequestMethod.currentText())

        use_global_cookie_jar = self.mainWindow.requesterUseGlobalCookieJar.isChecked(
        )
        replacements = self.build_replacements(method, url)
        (method, url, headers,
         body) = self.process_template(url, templateText, replacements)

        sequenceId = None
        if self.mainWindow.requesterSequenceCheckBox.isChecked():
            sequenceId = str(
                self.mainWindow.requesterSequenceComboBox.itemData(
                    self.mainWindow.requesterSequenceComboBox.currentIndex()))
        self.requestRunner = RequestRunner(self.framework, self)
        if use_global_cookie_jar:
            self.requesterCookieJar = self.framework.get_global_cookie_jar()
        else:
            self.requesterCookieJar = InMemoryCookieJar(self.framework, self)

        self.requestRunner.setup(self.requester_response_received,
                                 self.requesterCookieJar, sequenceId)

        self.pending_request = self.requestRunner.queue_request(
            method, url, headers, body)
        self.mainWindow.requesterSendButton.setText('Cancel')
        self.miniResponseRenderWidget.clear_response_render()

    def requester_response_received(self, response_id, context):
        if 0 != response_id:
            row = self.Data.read_responses_by_id(self.cursor, response_id)
            if row:
                response_item = interface.data_row_to_response_items(row)
                self.Data.insert_requester_history(self.cursor, response_id)
                self.requesterHistoryDataModel.append_data([response_item])

                url = response_item[ResponsesTable.URL]
                req_headers = response_item[ResponsesTable.REQ_HEADERS]
                req_body = response_item[ResponsesTable.REQ_DATA]
                res_headers = response_item[ResponsesTable.RES_HEADERS]
                res_body = response_item[ResponsesTable.RES_DATA]
                res_content_type = response_item[
                    ResponsesTable.RES_CONTENT_TYPE]

                self.miniResponseRenderWidget.populate_response_content(
                    url, req_headers, req_body, res_headers, res_body,
                    res_content_type)

        self.mainWindow.requesterSendButton.setText('Send')
        self.pending_request = None

    def requester_bulk_request_button_clicked(self):
        if 'Cancel' == self.mainWindow.bulkRequestPushButton.text(
        ) and self.pending_bulk_requests is not None:
            self.cancel_bulk_requests = True
            for context, pending_request in self.pending_bulk_requests.items():
                pending_request.cancel()
            self.pending_bulk_requests = None
            self.mainWindow.bulkRequestPushButton.setText('Send')
            self.mainWindow.bulkRequestProgressBar.setValue(0)
            return

        if self.pending_bulk_requests is None:
            self.pending_bulk_requests = {}

        method = str(self.mainWindow.bulkRequestMethodEdit.currentText())
        templateText = str(
            self.mainWindow.bulkRequestTemplateEdit.toPlainText())

        template_url = str(self.mainWindow.bulkRequestUrlEdit.text())

        url_list = str(self.mainWindow.bulkRequestUrlListEdit.toPlainText())
        self.framework.set_raft_config_value('bulkRequestUrlListEdit',
                                             url_list)
        request_urls = url_list.splitlines()
        self.mainWindow.bulkRequestProgressBar.setValue(0)
        self.mainWindow.bulkRequestProgressBar.setMaximum(len(request_urls))

        sequenceId = None
        if self.mainWindow.bulkRequestSequenceCheckBox.isChecked():
            sequenceId = str(
                self.mainWindow.bulkRequestSequenceComboBox.itemData(
                    self.mainWindow.bulkRequestSequenceComboBox.currentIndex())
            )

        first = True
        self.cancel_bulk_requests = False
        for request_url in request_urls:
            if self.cancel_bulk_requests:
                break
            request_url = request_url.strip()
            if request_url:
                context = uuid.uuid4().hex
                # TODO: move this hack
                if '$' in template_url:
                    replacements = self.build_replacements(method, request_url)
                    url = self.re_replacement.sub(
                        lambda m: replacements.get(m.group(1)), template_url)
                else:
                    url = request_url

                if not self.scopeController.isUrlInScope(url, url):
                    self.framework.log_warning(
                        'skipping out of scope URL: %s' % (url))
                    self.mainWindow.bulkRequestProgressBar.setValue(
                        self.mainWindow.bulkRequestProgressBar.value() + 1)
                    continue

                use_global_cookie_jar = self.mainWindow.bulkRequestUseGlobalCookieJar.isChecked(
                )
                replacements = self.build_replacements(method, url)
                (method, url, headers,
                 body) = self.process_template(url, templateText, replacements)

                if first:
                    self.mainWindow.bulkRequestPushButton.setText('Cancel')
                    if use_global_cookie_jar:
                        self.bulkRequesterCookieJar = self.framework.get_global_cookie_jar(
                        )
                    else:
                        self.bulkRequesterCookieJar = InMemoryCookieJar(
                            self.framework, self)
                    self.bulk_requestRunner = RequestRunner(
                        self.framework, self)
                    self.bulk_requestRunner.setup(
                        self.requester_bulk_response_received,
                        self.bulkRequesterCookieJar, sequenceId)
                    first = False

                self.pending_bulk_requests[
                    context] = self.bulk_requestRunner.queue_request(
                        method, url, headers, body, context)

    def requester_bulk_response_received(self, response_id, context):
        self.mainWindow.bulkRequestProgressBar.setValue(
            self.mainWindow.bulkRequestProgressBar.value() + 1)
        context = str(context)
        if self.pending_bulk_requests is not None:
            try:
                self.pending_bulk_requests.pop(context)
            except KeyError as e:
                pass
        if 0 != response_id:
            row = self.Data.read_responses_by_id(self.cursor, response_id)
            if row:
                response_item = interface.data_row_to_response_items(row)
                self.Data.insert_requester_history(self.cursor, response_id)
                self.requesterHistoryDataModel.append_data([response_item])

        finished = False
        if self.pending_bulk_requests is None or len(
                self.pending_bulk_requests) == 0:
            self.mainWindow.bulkRequestProgressBar.setValue(
                self.mainWindow.bulkRequestProgressBar.maximum())
            finished = True
        elif self.mainWindow.bulkRequestProgressBar.value(
        ) == self.mainWindow.bulkRequestProgressBar.maximum():
            finished = True
        if finished:
            self.mainWindow.bulkRequestPushButton.setText('Send')

    def handle_sequenceRunnerRunButton_clicked(self):
        """ Run a sequence """
        if 'Cancel' == self.mainWindow.sequenceRunnerRunButton.text(
        ) and self.pending_sequence_requests is not None:
            self.cancel_sequence_requests = True
            for context, pending_request in self.pending_sequence_requests.items(
            ):
                pending_request.cancel()
            self.pending_sequence_requests = None
            self.mainWindow.sequenceRunnerButton.setText('Send')
            self.mainWindow.sequenceRunnerButton.setValue(0)
            return

        self.sequenceRunnerDataModel.clearModel()

        sequenceId = str(
            self.mainWindow.sequenceRunnerSequenceComboBox.itemData(
                self.mainWindow.sequenceRunnerSequenceComboBox.currentIndex()))
        use_global_cookie_jar = self.mainWindow.sequenceRunnerUseGlobalCookieJar.isChecked(
        )
        if use_global_cookie_jar:
            self.sequenceRunnerCookieJar = self.framework.get_global_cookie_jar(
            )
        else:
            self.sequenceRunnerCookieJar = InMemoryCookieJar(
                self.framework, self)

        self.sequence_requestRunner = RequestRunner(self.framework, self)
        self.sequence_requestRunner.setup(
            self.sequence_runner_response_received,
            self.sequenceRunnerCookieJar, sequenceId)
        self.pending_sequence_requests = self.sequence_requestRunner.run_sequence(
        )
        self.mainWindow.sequenceRunnerRunButton.setText('Cancel')

    def sequence_runner_response_received(self, response_id, context):
        context = str(context)
        if self.pending_sequence_requests is not None:
            try:
                self.pending_sequence_requests.pop(context)
            except KeyError as e:
                print((e))
                pass

        if 0 != response_id:
            row = self.Data.read_responses_by_id(self.cursor, response_id)
            if row:
                response_item = interface.data_row_to_response_items(row)
                self.sequenceRunnerDataModel.append_data([response_item])

        if self.pending_sequence_requests is None or len(
                self.pending_sequence_requests) == 0:
            self.mainWindow.sequenceRunnerRunButton.setText('Send')

    def requester_history_clear_button_clicked(self):
        self.Data.clear_requester_history(self.cursor)
        self.requesterHistoryDataModel.clearModel()

    def fill_sequences_combo_box(self, comboBox):
        selectedText = comboBox.currentText()

        comboBox.clear()
        for row in self.Data.get_all_sequences(self.cursor):
            sequenceItem = [m or '' for m in row]
            name = str(sequenceItem[1])
            Id = str(sequenceItem[0])
            item = comboBox.addItem(name, Id)

        if selectedText:
            index = comboBox.findText(selectedText)
            if index != -1:
                comboBox.setCurrentIndex(index)

    def build_replacements(self, method, url):
        replacements = {}
        splitted = urlparse.urlsplit(url)
        replacements['method'] = method.upper()
        replacements['url'] = url
        replacements['scheme'] = splitted.scheme or ''
        replacements['netloc'] = splitted.netloc or ''
        replacements['host'] = splitted.hostname or ''
        replacements['path'] = splitted.path or '/'
        replacements['query'] = splitted.query or ''
        replacements['fragment'] = splitted.fragment or ''
        replacements['request_uri'] = urlparse.urlunsplit(
            ('', '', replacements['path'], replacements['query'], ''))
        replacements['user_agent'] = self.framework.useragent()
        return replacements

    def process_template(self, url, template, replacements):

        method, uri = '', ''
        headers, body = '', ''

        # TODO: this allows for missing entries -- is this good?
        func = lambda m: replacements.get(m.group(1))

        prev = 0
        while True:
            n = template.find('\n', prev)
            if -1 == n:
                break
            if n > 0 and '\r' == template[n - 1]:
                line = template[prev:n - 1]
            else:
                line = template[prev:n]

            if 0 == len(line):
                # end of headers
                headers = template[0:n + 1]
                body = template[n + 1:]
                break
            prev = n + 1

        if not headers:
            headers = template
            body = ''

        # TODO: could work from ordered dict to main order?
        headers_dict = {}
        first = True
        for line in headers.splitlines():
            if not line:
                break
            if '$' in line:
                line = self.re_replacement.sub(func, line)
            if first:
                m = self.re_request.match(line)
                if not m:
                    raise Exception(
                        'Invalid HTTP request: failed to match request line: %s'
                        % (line))
                method = m.group(1)
                uri = m.group(2)
                first = False
                continue

            if ':' in line:
                name, value = [v.strip() for v in line.split(':', 1)]
                headers_dict[name] = value

        if '$' in body:
            body = self.re_replacement.sub(func, body)

        url = urlparse.urljoin(url, uri)

        return (method, url, headers_dict, body)
示例#8
0
class SearchDialog(QDialog, SearchDialog.Ui_SearchDialog):
    """ The search dialog """
    def __init__(self, framework, parent=None):
        super(SearchDialog, self).__init__(parent)
        self.setupUi(self)

        self.connect(self, SIGNAL('finished()'), self.finishedHandler)

        self.framework = framework

        # progress dialog
        self.Progress = ProgressDialog(self)
        QObject.connect(self.Progress, SIGNAL('canceled()'),
                        self.canceledHandler)

        self.searchRequestResponse = RequestResponseWidget(
            self.framework, self.searchTabWidget,
            self.searchSearchControlPlaceholder, self)

        self.searchResultsModel = ResponsesDataModel.ResponsesDataModel(
            self.framework, self)
        self.searchResultsTree.setModel(self.searchResultsModel)
        self.searchResultsTree.clicked.connect(self.fill_bottom)
        self.searchResultsTree.doubleClicked.connect(
            self.response_item_double_clicked)
        self.searchResultsTree.setSortingEnabled(True)
        self.searchResultsTree.sortByColumn(0, Qt.AscendingOrder)

        self.thread = SearchThread(self.framework, self.searchResultsModel)
        self.thread.start(QThread.LowestPriority)
        self.finished.connect(self.finishedHandler)
        self.searchPushButton.pressed.connect(self.startSearch)
        self.connect(self, SIGNAL('searchFinished()'),
                     self.searchFinishedHandler, Qt.QueuedConnection)
        # Create context menu
        self.resultsContextMenu = ResponsesContextMenuWidget(
            self.framework, self.searchResultsModel, self.searchResultsTree,
            self)
        self.resultsContextMenu.set_currentChanged_callback(self.fill_bottom)

    def response_item_double_clicked(self, index):
        Id = interface.index_to_id(self.searchResultsModel, index)
        if Id:
            dialog = RequestResponseDetailDialog(self.framework, str(Id), self)
            dialog.show()
            dialog.exec_()

    def fill_bottom(self, index):
        if self.searchFilling:
            return
        Id = interface.index_to_id(self.searchResultsModel, index)
        if Id:
            self.searchRequestResponse.fill(str(Id))
            self.searchRequestResponse.set_search_info(
                str(self.searchText.text()).strip(),
                self.cbRegularExpression.isChecked())

    def startSearch(self):
        text = str(self.searchText.text()).strip()
        if 0 == len(text):
            return
        options = {
            'CaseSensitive': self.cbCaseSensitive.isChecked(),
            'RegularExpression': self.cbRegularExpression.isChecked(),
            'InvertSearch': self.cbInvertSearch.isChecked(),
            'Wildcard': self.cbWildcard.isChecked(),
        }
        locations = {
            'RequestHeaders': self.cbRequestHeaders.isChecked(),
            'RequestBody': self.cbRequestBody.isChecked(),
            'ResponseHeaders': self.cbResponseHeaders.isChecked(),
            'ResponseBody': self.cbResponseBody.isChecked(),
            'RequestUrl': self.cbRequestUrl.isChecked(),
            'AnalystNotes': self.cbAnalystNotes.isChecked(),
        }
        searchCriteria = SearchCriteria(text, options, locations)
        self.Progress.show()
        self.searchFilling = True
        self.searchRequestResponse.clear()
        self.searchResultsTree.clearSelection()
        self.searchResultsModel.clearModel()
        self.thread.startSearch(searchCriteria, self)

    def searchFinishedHandler(self):
        self.Progress.reset()
        self.searchFilling = False

    def finishedHandler(self, code):
        self.thread.emit(SIGNAL('quit()'))

    def canceledHandler(self):
        if self.Progress.wasCanceled():
            self.thread.stopSearch()

    def finishedHandler(self):
        self.thread.quit()
示例#9
0
class SearchDialog(QDialog, SearchDialog.Ui_SearchDialog):
    """ The search dialog """
    
    def __init__(self, framework, parent=None):
        super(SearchDialog, self).__init__(parent)
        self.setupUi(self)

        self.connect(self, SIGNAL('finished()'), self.finishedHandler)

        self.framework = framework

        # progress dialog
        self.Progress = ProgressDialog(self)
        QObject.connect(self.Progress, SIGNAL('canceled()'), self.canceledHandler)

        self.searchRequestResponse = RequestResponseWidget(self.framework, self.searchTabWidget, self.searchSearchControlPlaceholder, self)

        self.searchResultsModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
        self.searchResultsTree.setModel(self.searchResultsModel)
        self.searchResultsTree.clicked.connect(self.fill_bottom)
        self.searchResultsTree.doubleClicked.connect(self.response_item_double_clicked)
        self.searchResultsTree.setSortingEnabled(True)
        self.searchResultsTree.sortByColumn(0, Qt.AscendingOrder)

        self.thread = SearchThread(self.framework, self.searchResultsModel)
        self.thread.start(QThread.LowestPriority)
        self.finished.connect(self.finishedHandler)
        self.searchPushButton.pressed.connect(self.startSearch)
        self.connect(self, SIGNAL('searchFinished()'), self.searchFinishedHandler, Qt.QueuedConnection)
        # Create context menu
        self.resultsContextMenu = ResponsesContextMenuWidget(self.framework, self.searchResultsModel, self.searchResultsTree, self)
        self.resultsContextMenu.set_currentChanged_callback(self.fill_bottom)

    def response_item_double_clicked(self, index):
        Id = interface.index_to_id(self.searchResultsModel, index)
        if Id:
            dialog = RequestResponseDetailDialog(self.framework, str(Id), self)
            dialog.show()
            dialog.exec_()

    def fill_bottom(self, index):
        if self.searchFilling:
            return
        Id = interface.index_to_id(self.searchResultsModel, index)
        if Id:
            self.searchRequestResponse.fill(str(Id))
            self.searchRequestResponse.set_search_info(str(self.searchText.text()).strip(), self.cbRegularExpression.isChecked())

    def startSearch(self):
        text = str(self.searchText.text()).strip()
        if 0 == len(text):
            return
        options = {
            'CaseSensitive' : self.cbCaseSensitive.isChecked(),
            'RegularExpression' : self.cbRegularExpression.isChecked(),
            'InvertSearch' : self.cbInvertSearch.isChecked(),
            'Wildcard' : self.cbWildcard.isChecked(),
            }
        locations = {
            'RequestHeaders' : self.cbRequestHeaders.isChecked(),
            'RequestBody' : self.cbRequestBody.isChecked(),
            'ResponseHeaders' : self.cbResponseHeaders.isChecked(),
            'ResponseBody' : self.cbResponseBody.isChecked(),
            'RequestUrl' : self.cbRequestUrl.isChecked(),
            'AnalystNotes' : self.cbAnalystNotes.isChecked(),
            }
        searchCriteria = SearchCriteria(text, options, locations)
        self.Progress.show()
        self.searchFilling = True
        self.searchRequestResponse.clear()
        self.searchResultsTree.clearSelection()
        self.searchResultsModel.clearModel()
        self.thread.startSearch(searchCriteria, self)

    def searchFinishedHandler(self):
        self.Progress.reset()
        self.searchFilling = False

    def finishedHandler(self, code):
        self.thread.emit(SIGNAL('quit()'))

    def canceledHandler(self):
        if self.Progress.wasCanceled():
            self.thread.stopSearch()
    
    def finishedHandler(self):
        self.thread.quit()
示例#10
0
    def setup_others(self):

        # set request response factory
        self.framework.setRequestResponseFactory(RequestResponseFactory.RequestResponseFactory(self.framework, self))

        # scoping and spider
        self.framework.setScopeController(ScopeController.ScopeController(self.framework, self))
        self.framework.setSpiderConfig(SpiderConfig.SpiderConfig(self.framework, self))

        # setup network accessmanager
        self.dbNetworkAccessManager = DatabaseNetworkAccessManager(self.framework, self.framework.get_global_cookie_jar())
        self.framework.setNetworkAccessManager(self.dbNetworkAccessManager)

        # set up tabs
        self.vulnerabilitiesTab = VulnerabilitiesTab.VulnerabilitiesTab(self.framework, self)
        self.cookiesTab = CookiesTab.CookiesTab(self.framework, self)
        self.requesterTab = RequesterTab.RequesterTab(self.framework, self)
        self.webfuzzerTab = WebFuzzerTab.WebFuzzerTab(self.framework, self)
        self.domFuzzerTab = DomFuzzerTab.DomFuzzerTab(self.framework, self)
        self.crawlerTab = CrawlerTab.CrawlerTab(self.framework, self)
        self.encoderTab = EncoderTab.EncoderTab(self.framework, self)
        self.scopingTab = ScopingTab.ScopingTab(self.framework, self)
        self.testerTab = TesterTab.TesterTab(self.framework, self)
        self.logTab = LogTab.LogTab(self.framework, self)
        self.quickAnalysisTab = QuickAnalysisTab.QuickAnalysisTab(self.framework, self)

        # sitemap
        self.siteMapRequestResponse = RequestResponseWidget(self.framework, self.sitemapTabPlaceholder, self.sitemapSearchControlPlaceholder, self)

        # TODO: cleanup site map and optimize...
        self.siteMapModel = SiteMapModel.SiteMapModel(self.framework)
        self.siteMapThread = SiteMapThread.SiteMapThread(self.framework, self.siteMapModel, self)
        self.siteMapThread.start(QThread.LowestPriority)

        self.importerThread = ImporterThread.ImporterThread(self.framework, self)
        self.connect(self, SIGNAL('runImportFinished()'), self.import_file_finished, Qt.QueuedConnection)
        self.importerThread.start(QThread.LowestPriority)

        self.treeViewSitemap.setContextMenuPolicy(Qt.CustomContextMenu)
        self.treeViewSitemapMenu = QMenu(self)
        treeViewSitemapCopyUrlAction = QAction("Copy URL", self)
        treeViewSitemapCopyUrlAction.triggered.connect(self.sitemap_copy_url)
        self.treeViewSitemapMenu.addAction(treeViewSitemapCopyUrlAction)

        self.connect(self.treeViewSitemap, SIGNAL('customContextMenuRequested(const QPoint &)'), self.sitemap_context_menu) 
        self.treeViewSitemap.activated.connect(self.siteMapRequestResponse.viewItemSelected)
        # TODO: clicked is annoying sometimes
        self.treeViewSitemap.clicked.connect(self.siteMapRequestResponse.viewItemSelected)
        self.treeViewSitemap.setModel(self.siteMapModel)

        # view tab
        self.viewTabRequestResponse = RequestResponseWidget(self.framework, self.responseTabPlaceholder, self.responseSearchControlPlaceholder, self)

        # Responses data
        self.responsesDataModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
        self.responsesDataTree.setModel(self.responsesDataModel)
        self.responsesThread = ResponsesThread.ResponsesThread(self.framework, self.responsesDataModel, self)
        self.responsesThread.start(QThread.LowPriority)
        self.connect(self, SIGNAL('fillResponsesFinished()'), self.fillResponsesFinishedHandler, Qt.QueuedConnection)
        self.responsesContextMenu = ResponsesContextMenuWidget(self.framework, self.responsesDataModel, self.responsesDataTree, self)
        self.responsesContextMenu.set_currentChanged_callback(self.fill_bottom)

        #analysis results tab
        #add response widget to the bottom
        self.mainAnalysisTabRequestResponse = RequestResponseWidget(self.framework, self.mainAnalysisTabPlaceholder, self.mainAnalysisSearchControlPlaceholder, self)

        self.analyzerThread = AnalyzerThread.AnalyzerThread(self.framework, self)
        self.connect(self, SIGNAL('runAnalysisFinished(QString)'), self.handle_runAnalysisFinished, Qt.QueuedConnection)
        self.analyzerThread.start(QThread.LowestPriority)

        # dom fuzzer thread
        self.domFuzzerQueueDataModel = DomFuzzerQueueDataModel.DomFuzzerQueueDataModel(self.framework, self)
        self.domFuzzerFuzzQueueTable.setModel(self.domFuzzerQueueDataModel)
        self.domFuzzerResultsDataModel = DomFuzzerResultsDataModel.DomFuzzerResultsDataModel(self.framework, self)
        self.domFuzzerResultsTreeView.setModel(self.domFuzzerResultsDataModel)
        self.domFuzzerThread = DomFuzzerThread.DomFuzzerThread(self.framework, self.domFuzzerQueueDataModel, self.domFuzzerResultsDataModel,  self)
        self.domFuzzerThread.start(QThread.LowestPriority)
        self.domFuzzerTab.set_fuzzer_thread(self.domFuzzerThread)
        self.domFuzzerTab.setup_fuzzer_results_treeview()

        # spider thread
        self.spiderQueueDataModel = SpiderQueueDataModel.SpiderQueueDataModel(self.framework, self)
        self.crawlerSpiderQueueTreeView.setModel(self.spiderQueueDataModel)
        self.spiderPendingResponsesDataModel = SpiderPendingResponsesDataModel.SpiderPendingResponsesDataModel(self.framework, self)
        self.crawlerSpiderPendingResponsesTreeView.setModel(self.spiderPendingResponsesDataModel)
        self.spiderPendingAnalysisDataModel = SpiderPendingAnalysisDataModel.SpiderPendingAnalysisDataModel(self.framework, self)
        self.crawlerSpiderPendingAnalysisTreeView.setModel(self.spiderPendingAnalysisDataModel)
        self.spiderInternalStateDataModel = SpiderInternalStateDataModel.SpiderInternalStateDataModel(self.framework, self)
        self.crawlerSpiderInternalStateTreeView.setModel(self.spiderInternalStateDataModel)
        self.spiderThread = SpiderThread.SpiderThread(self.framework, self.spiderQueueDataModel, self.spiderPendingResponsesDataModel, self.spiderPendingAnalysisDataModel, self.spiderInternalStateDataModel, self)
        self.spiderThread.start(QThread.LowestPriority)
        self.crawlerTab.set_spider_thread(self.spiderThread)

        # quick analysis thread
        self.quickAnalysisThread = QuickAnalysisThread.QuickAnalysisThread(self.framework, self)
        self.quickAnalysisThread.start(QThread.LowestPriority)
        self.quickAnalysisTab.set_quick_analysis_thread(self.quickAnalysisThread)

        # handlers
        self.framework.register_browser_openers(self.open_url_in_browser, self.open_content_in_browser)

        self.do_db_connect()
示例#11
0
class RaftMain(QMainWindow, RaftMain.Ui_MainWindow):
    """ Reimplementation of the imported Ui_MainWindow class """
    def __init__(self, dbfilename = '', parent=None):
        super(RaftMain, self).__init__(parent)
        if MAC:
            qt_mac_set_native_menubar(False)
        self.setupUi(self)

        # hide currently unimplemented features
        self.reqTabRawRequestTab.hide()
        self.reqTabRawRequestTab.setParent(None)
        self.wfUnusedSequenceTab.hide()
        self.wfUnusedSequenceTab.setParent(None)

        # initialize framework
        self.framework = Framework(self)

        # default filename is temp.raftdb
        if dbfilename:
            self.dbfilename = dbfilename
        else:
            self.dbfilename = self.framework.get_temp_db_filename()

        # restore settings
        self.restore_settings()

        # Create progress dialog
        self.Progress = ProgressDialog()

        # add helper and utility singletons
        # TODO: should be base extractor and that loads/returns appropriate type
        self.contentExtractor = BaseExtractor.BaseExtractor()
        self.framework.setContentExtractor(self.contentExtractor)

        # Create actions for items
        self.responsesDataTree.doubleClicked.connect(self.response_item_double_clicked)
        self.fillingDataTree = False
        self.responsesDataTree.clicked.connect(self.fill_bottom)
        self.responsesDataTree.activated.connect(self.fill_bottom)
        self.responsesDataTree.setSortingEnabled(True)
        self.responsesDataTree.sortByColumn(0, Qt.AscendingOrder)
        
        #analysis tab connections
        self.mainAnalysisTreeWidget.clicked.connect(self.analysistree_handle_click)
        self.mainAnalysisTreeWidget.activated.connect(self.analysistree_handle_click)
        self.mainAnalysisTreeWidget.expanded.connect(self.analysistree_handle_expand)
        
        self.cookiesTabIndex = self.mainTabWidget.indexOf(self.tabMainCookies)
        self.mainTabWidget.currentChanged.connect(self.main_tab_change)
        
        # Toolbar buttons and actions
        self.actionButtonOpen.triggered.connect(self.open_file)
        self.actionZoomIn.triggered.connect(self.zoom_in)
        self.actionZoomOut.triggered.connect(self.zoom_out)
        self.actionDiff.triggered.connect(self.diff)
        self.actionAnalyze.triggered.connect(self.analyze_content)
        self.actionSequence.triggered.connect(self.display_sequence)
        self.actionConfig.triggered.connect(self.display_config)
        self.actionSearch.triggered.connect(self.display_search)
        self.actionBrowser.triggered.connect(self.launch_browser)
        
        # Create the actions for the buttons
        # self.connect(self.encodeButton, SIGNAL("clicked()"), self.encode_values)
        # self.connect(self.encodeWrapButton, SIGNAL("clicked()"), self.wrap_encode)
        # self.connect(self.decodeButton, SIGNAL("clicked()"), self.decode_values)
        # self.connect(self.decodeWrapButton, SIGNAL("clicked()"), self.wrap_decode)
        
        # Actions for Menus
        self.actionNew_Project.triggered.connect(self.new_project)
        self.actionOpen.triggered.connect(self.open_file)
        self.actionSave_As.triggered.connect(self.save_as)
        self.actionImport_RaftCaptureXml.triggered.connect(lambda x: self.import_raft('raft_capture_xml'))
        self.actionImport_BurpLog.triggered.connect(lambda x: self.import_burp('burp_log'))
        self.actionImport_BurpState.triggered.connect(lambda x: self.import_burp('burp_state'))
        self.actionImport_BurpXml.triggered.connect(lambda x: self.import_burp('burp_xml'))
        self.actionImport_Burp_Vuln_XML.triggered.connect(lambda x: self.import_burp('burp_vuln_xml'))
        self.actionImport_AppScan_XML.triggered.connect(lambda x: self.import_appscan('appscan_xml'))
        self.actionImport_WebScarab.triggered.connect(self.import_webscarab)
        self.actionImport_ParosMessages.triggered.connect(lambda x: self.import_paros('paros_message'))
        self.actionRefresh_Responses.triggered.connect(self.refresh_responses)
        self.actionClear_Responses.triggered.connect(self.clear_responses)
        self.actionExport_Settings.triggered.connect(self.export_settings)
        self.actionImport_Settings.triggered.connect(self.import_settings)

        self.actionEncoder_Decoder.triggered.connect(self.detach_encoder)

        # Actions for configuration
        self.actionConfiguration_BlackHoleNetwork.triggered.connect(lambda x: self.raft_config_toggle('black_hole_network', x))
        self.actionAbout_RAFT.triggered.connect(self.display_about)
        self.actionAnalysisConfiguration.triggered.connect(self.display_analysis_config)

        # Declare, but do not fill, list of analyzers
        self.analyzerlist = AnalyzerList(self.framework)

        self.setup_others()

    def db_attach(self):
        self.framework.debug_log('Database %s attached' % (self.db))
        self.cursor = self.Data.allocate_thread_cursor()

    def db_detach(self):
        self.framework.debug_log('Database %s detached' % (self.db))
        if self.Data:
            self.close_cursor()

    def close_cursor(self):
        if self.cursor and self.Data:
            self.cursor.close()
            self.Data.release_thread_cursor(self.cursor)
            self.cursor = None

    def do_db_connect(self):
        ###
        # Set up initial data connections for tools
        ###
        self.framework.subscribe_database_events(self.db_attach, self.db_detach)

        self.Progress.show()
        # Set initial temp.raftdb file for storage of imported data.
        self.Data = database.Db(__version__, self.framework.report_exception)
        self.db = self.dbfilename
        self.databaseThread = DatabaseThread.DatabaseThread(self.framework, self.Data, self)
        self.connect(self, SIGNAL('connectDbFinished()'), self.connectDbFinishedHandler, Qt.QueuedConnection)
        self.databaseThread.start()
        self.databaseThread.connectDb(self.db, self)

    def connectDbFinishedHandler(self):
        self.framework.setDB(self.Data, self.db)
        self.actionConfiguration_BlackHoleNetwork.setChecked(self.framework.get_raft_config_value('black_hole_network', bool))
        self.Progress.close()
        self.refresh_analysis_tab()

    def setup_others(self):

        # set request response factory
        self.framework.setRequestResponseFactory(RequestResponseFactory.RequestResponseFactory(self.framework, self))

        # scoping and spider
        self.framework.setScopeController(ScopeController.ScopeController(self.framework, self))
        self.framework.setSpiderConfig(SpiderConfig.SpiderConfig(self.framework, self))

        # setup network accessmanager
        self.dbNetworkAccessManager = DatabaseNetworkAccessManager(self.framework, self.framework.get_global_cookie_jar())
        self.framework.setNetworkAccessManager(self.dbNetworkAccessManager)

        # set up tabs
        self.vulnerabilitiesTab = VulnerabilitiesTab.VulnerabilitiesTab(self.framework, self)
        self.cookiesTab = CookiesTab.CookiesTab(self.framework, self)
        self.requesterTab = RequesterTab.RequesterTab(self.framework, self)
        self.webfuzzerTab = WebFuzzerTab.WebFuzzerTab(self.framework, self)
        self.domFuzzerTab = DomFuzzerTab.DomFuzzerTab(self.framework, self)
        self.crawlerTab = CrawlerTab.CrawlerTab(self.framework, self)
        self.encoderTab = EncoderTab.EncoderTab(self.framework, self)
        self.scopingTab = ScopingTab.ScopingTab(self.framework, self)
        self.testerTab = TesterTab.TesterTab(self.framework, self)
        self.logTab = LogTab.LogTab(self.framework, self)
        self.quickAnalysisTab = QuickAnalysisTab.QuickAnalysisTab(self.framework, self)

        # sitemap
        self.siteMapRequestResponse = RequestResponseWidget(self.framework, self.sitemapTabPlaceholder, self.sitemapSearchControlPlaceholder, self)

        # TODO: cleanup site map and optimize...
        self.siteMapModel = SiteMapModel.SiteMapModel(self.framework)
        self.siteMapThread = SiteMapThread.SiteMapThread(self.framework, self.siteMapModel, self)
        self.siteMapThread.start(QThread.LowestPriority)

        self.importerThread = ImporterThread.ImporterThread(self.framework, self)
        self.connect(self, SIGNAL('runImportFinished()'), self.import_file_finished, Qt.QueuedConnection)
        self.importerThread.start(QThread.LowestPriority)

        self.treeViewSitemap.setContextMenuPolicy(Qt.CustomContextMenu)
        self.treeViewSitemapMenu = QMenu(self)
        treeViewSitemapCopyUrlAction = QAction("Copy URL", self)
        treeViewSitemapCopyUrlAction.triggered.connect(self.sitemap_copy_url)
        self.treeViewSitemapMenu.addAction(treeViewSitemapCopyUrlAction)

        self.connect(self.treeViewSitemap, SIGNAL('customContextMenuRequested(const QPoint &)'), self.sitemap_context_menu) 
        self.treeViewSitemap.activated.connect(self.siteMapRequestResponse.viewItemSelected)
        # TODO: clicked is annoying sometimes
        self.treeViewSitemap.clicked.connect(self.siteMapRequestResponse.viewItemSelected)
        self.treeViewSitemap.setModel(self.siteMapModel)

        # view tab
        self.viewTabRequestResponse = RequestResponseWidget(self.framework, self.responseTabPlaceholder, self.responseSearchControlPlaceholder, self)

        # Responses data
        self.responsesDataModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
        self.responsesDataTree.setModel(self.responsesDataModel)
        self.responsesThread = ResponsesThread.ResponsesThread(self.framework, self.responsesDataModel, self)
        self.responsesThread.start(QThread.LowPriority)
        self.connect(self, SIGNAL('fillResponsesFinished()'), self.fillResponsesFinishedHandler, Qt.QueuedConnection)
        self.responsesContextMenu = ResponsesContextMenuWidget(self.framework, self.responsesDataModel, self.responsesDataTree, self)
        self.responsesContextMenu.set_currentChanged_callback(self.fill_bottom)

        #analysis results tab
        #add response widget to the bottom
        self.mainAnalysisTabRequestResponse = RequestResponseWidget(self.framework, self.mainAnalysisTabPlaceholder, self.mainAnalysisSearchControlPlaceholder, self)

        self.analyzerThread = AnalyzerThread.AnalyzerThread(self.framework, self)
        self.connect(self, SIGNAL('runAnalysisFinished(QString)'), self.handle_runAnalysisFinished, Qt.QueuedConnection)
        self.analyzerThread.start(QThread.LowestPriority)

        # dom fuzzer thread
        self.domFuzzerQueueDataModel = DomFuzzerQueueDataModel.DomFuzzerQueueDataModel(self.framework, self)
        self.domFuzzerFuzzQueueTable.setModel(self.domFuzzerQueueDataModel)
        self.domFuzzerResultsDataModel = DomFuzzerResultsDataModel.DomFuzzerResultsDataModel(self.framework, self)
        self.domFuzzerResultsTreeView.setModel(self.domFuzzerResultsDataModel)
        self.domFuzzerThread = DomFuzzerThread.DomFuzzerThread(self.framework, self.domFuzzerQueueDataModel, self.domFuzzerResultsDataModel,  self)
        self.domFuzzerThread.start(QThread.LowestPriority)
        self.domFuzzerTab.set_fuzzer_thread(self.domFuzzerThread)
        self.domFuzzerTab.setup_fuzzer_results_treeview()

        # spider thread
        self.spiderQueueDataModel = SpiderQueueDataModel.SpiderQueueDataModel(self.framework, self)
        self.crawlerSpiderQueueTreeView.setModel(self.spiderQueueDataModel)
        self.spiderPendingResponsesDataModel = SpiderPendingResponsesDataModel.SpiderPendingResponsesDataModel(self.framework, self)
        self.crawlerSpiderPendingResponsesTreeView.setModel(self.spiderPendingResponsesDataModel)
        self.spiderPendingAnalysisDataModel = SpiderPendingAnalysisDataModel.SpiderPendingAnalysisDataModel(self.framework, self)
        self.crawlerSpiderPendingAnalysisTreeView.setModel(self.spiderPendingAnalysisDataModel)
        self.spiderInternalStateDataModel = SpiderInternalStateDataModel.SpiderInternalStateDataModel(self.framework, self)
        self.crawlerSpiderInternalStateTreeView.setModel(self.spiderInternalStateDataModel)
        self.spiderThread = SpiderThread.SpiderThread(self.framework, self.spiderQueueDataModel, self.spiderPendingResponsesDataModel, self.spiderPendingAnalysisDataModel, self.spiderInternalStateDataModel, self)
        self.spiderThread.start(QThread.LowestPriority)
        self.crawlerTab.set_spider_thread(self.spiderThread)

        # quick analysis thread
        self.quickAnalysisThread = QuickAnalysisThread.QuickAnalysisThread(self.framework, self)
        self.quickAnalysisThread.start(QThread.LowestPriority)
        self.quickAnalysisTab.set_quick_analysis_thread(self.quickAnalysisThread)

        # handlers
        self.framework.register_browser_openers(self.open_url_in_browser, self.open_content_in_browser)

        self.do_db_connect()

    def fillResponses(self, fillAll = False):
        self.Progress.show()
        self.fillingDataTree = True
        self.fillAll = fillAll
        self.responsesThread.fillResponses(fillAll, self)

    def fillResponsesFinishedHandler(self):
        self.Progress.close()
        self.fillingDataTree = False

        self.siteMapThread.populateSiteMap(self.fillAll)
        
    def restore_settings(self):
        # TODO: make constants make sense
        settings = QSettings('RaftDev', 'Raft');
        saved = settings.value('RaftMain/geometry')
        if saved is not None:
            self.restoreGeometry(saved);

    def closeEvent(self, event):
        settings = QSettings('RaftDev', 'Raft');
        settings.setValue('RaftMain/geometry', self.saveGeometry());
        QWidget.closeEvent(self, event)

    def create_file(self):
        """ Open a dialog and allow for specifying a filename as well as creating a new database """
        
        file = QFileDialog.getSaveFileName(None, "Create File", "")
        self.Data.create_raft_db(str(file), __version__)

    def new_project(self):
        # Support '.raftdb' file types
        file = QFileDialog.getSaveFileName(None, "Create new RAFT DB file", "", "RAFT Database File (*.raftdb)")

        if file:
            self.Progress.show()
            try:
                # Reinitialize with the database value set from new db name
                self.framework.closeDB()

                # 2 seconds to settle
                QThread.sleep(2) 

                self.db = str(file)
                self.databaseThread.connectDb(self.db, self)
            finally:
                self.Progress.close()

    def open_file(self):
        """ Open File from file open dialog """
        
        file = QFileDialog.getOpenFileName(None, "Open file", "")
            
        if file != "":
            origdb = self.db
            self.Progress.show()
            # Reinitialize with the database value set
            self.framework.closeDB()
            self.db = str(file)
            self.databaseThread.connectDb(self.db, self)
        else:
            pass
        
    def save_as(self):
        """ Dialog and logic to save the temp.raftdb file to a working database """
        
        # Support '.db' file types
        file = QFileDialog.getSaveFileName(None, "Save to RAFT DB file", "", "RAFT Database File (*.raftdb)")

        if file:
            self.Progress.show()
            try:
                # Reinitialize with the database value set from new db name
                self.framework.closeDB()

                # 5 seconds to settle
                QThread.sleep(2) 

                # New db location
                new_db = str(file)
                shutil.move(self.db, new_db)
                self.db = new_db

                self.databaseThread.connectDb(self.db, self)
            finally:
                self.Progress.close()

    def refresh_responses(self):
        self.fillResponses(True)

    def clear_responses(self):
        response = self.display_confirm_dialog('Clear All Responses?\n\nAll response data will be permanently removed from the project database!')
        if response:
            self.Progress.show()
            try:
                # Truncate existing response values
                self.Data.truncate_response_data(self.cursor)
                self.framework.closeDB()
                self.databaseThread.connectDb(self.db, self)
            finally:
                self.Progress.close()

    def export_settings(self):
        filename = QFileDialog.getSaveFileName(None, "Export RAFT Settings", "", "RAFT Settings File (*.raftsettings)")
        if filename:
            self.Progress.show()
            SettingsFiles.process_export(self.framework, filename)
            self.Progress.close()

    def import_settings(self):
        filename = QFileDialog.getOpenFileName(None, "Import RAFT Settings", "", "RAFT Settings File (*.raftsettings)")
        if filename:
            self.Progress.show()
            SettingsFiles.process_import(self.framework, filename)
            self.Progress.close()
        
    def import_file_finished(self):
        self.Progress.close()
        self.fillResponses()
        
    def import_proxy_file(self, proxy_file, source):
        self.Progress.show()
        self.importerThread.runImport(importers, proxy_file, source, self)

    def import_proxy_files(self, proxy_filelist, source):
        self.Progress.show()
        self.importerThread.runImportList(importers, proxy_filelist, source, self)

    def import_burp(self, source):
        """ Import a Burp proxy log """
        files = QFileDialog.getOpenFileNames(None, "Open file", "")
        if files is not None:
            self.import_proxy_files(files, source)

    def import_appscan(self, source):
        """ Import an AppScan XML file """
        files = QFileDialog.getOpenFileNames(None, "Open file", "")
        if files is not None:
            self.import_proxy_files(files, source)

    def import_raft(self, source):
        """ Import a Raft Capture XML """
        files = QFileDialog.getOpenFileNames(None, "Open file", "")
        if files is not None:
            self.import_proxy_files(files, source)

    def import_webscarab(self, source):
        """ Import a WebScarab conversation log """

        # TODO: decide if there is a more friendly way to handle this
#        file = QFileDialog.getExistingDirectory(None, "Open saved conversation", "")
        file = QFileDialog.getOpenFileName(None, "Open saved conversation", "", "Converstation Log (conversationlog)")

        if file:
            self.import_proxy_file(str(file), 'webscarab')

    def import_paros(self, source):
        """ Import a Paros proxy log """
        
        file = QFileDialog.getOpenFileName(None, "Open file", "")

        if file:
            self.import_proxy_file(str(file), source)

################################################
# Is this section still being used now we are depend on the AnalyzerThread?  Can we rip it out?

    def analyze_content(self):
        """ Perform analysis on the captured content"""
        self.Progress.show()
        self.analyzerThread.runAnalysis(self)

    def handle_runAnalysisFinished(self, fullanalysistext = ''):
        self.Progress.close()
        #self.mainAnalysisEdit.setText(fullanalysistext)
        #self.refresh_analysis_tab()
        self.analysis_tab_add_results(None)

    def refresh_analysis_tab(self):
        self.runsdisplayed=list()
        
        rundata=self.Data.analysis_get_runs(self.cursor)
        self.populate_analysis_tree(rundata)
    
    def analysis_tab_add_results(self,results):
        rundata=self.Data.analysis_get_runs(self.cursor, lastx=1)
        self.populate_analysis_tree(rundata)
    
    def populate_analysis_tree(self,rundata):
        virtualparent=self.mainAnalysisTreeWidget.invisibleRootItem()
        resultfactory=ResultFactory()
        for run in rundata:
            runid=run[0]
            if runid not in self.runsdisplayed:
                self.runsdisplayed.append(runid)
                tempRun=AnalysisRun(run[1], run[2],resultfactory)
                tempRun.runid=runid
                tempRun.dbgenerated=True
                runtreeitem=tempRun.generateTreeItem(virtualparent)
                runtreeitem.customdata=tempRun
                tempRun.generateTreeChildren(self.Data,self.cursor,runtreeitem)

        
            
    def analysistree_handle_expand(self, index):
        item=self.mainAnalysisTreeView.itemFromIndex(index)
        item.wasexpanded=True
        self.analysistree_load_grandchildren(item)
                
    def analysistree_load_grandchildren(self, item):
        #For each child of this item:
        numchildren=item.childCount()
        for i in range(0,numchildren):
            childitem=item.child(i)
            childitem.wasexpanded=True
            #If the childitem doesn't have children, make them:
            if childitem.childCount()==0:
                childitem.customdata.generateTreeChildren(self.Data,self.cursor,childitem)
                childitem.wasexpanded=True

    def analysistree_handle_click(self, index):
        
        item=self.mainAnalysisTreeWidget.itemFromIndex(index)
        if hasattr(item,'customdata'):
            self.analysistree_load_decendants_to_bottom(item)
            self.mainAnalysisEdit.setText(item.customdata.toHTML())
        
            self.fill_analysis_request_response(item)
            
            if hasattr(item.customdata, 'getFoundData'):
                founddata=item.customdata.getFoundData()
                if founddata is not None:
                    self.set_analysis_request_response_highlight('response',founddata)

    def set_analysis_request_response_highlight(self, section, searchtext):
        self.mainAnalysisTabRequestResponse.set_search(section, searchtext)

    def analysistree_load_decendants_to_bottom(self,item):
        numchildren=item.childCount()
        if hasattr(item,'customdata') and numchildren==0: #(not hasattr(item,'wasexpanded') or not item.wasexpanded): 
            item.customdata.generateTreeChildren(self.Data, self.cursor, item)
            item.wasexpanded=True
        for i in range(0,numchildren):
            self.analysistree_load_decendants_to_bottom(item.child(i))
             
        

    def response_item_double_clicked(self, index):
        Id = interface.index_to_id(self.responsesDataModel, index)
        if Id:
            dialog = RequestResponseDetailDialog(self.framework, Id, self)
            dialog.show()
            dialog.exec_()

    def fill_bottom(self, index):
        """ Return the data from the database and fill bottom part of main window """
        if self.fillingDataTree:
            return
        Id = interface.index_to_id(self.responsesDataModel, index)
        if Id:
            self.viewTabRequestResponse.fill(str(Id))
    
    def fill_analysis_request_response(self, item):
        """ Return the data from the database and fill bottom part of main window """
        
        pageid=None

        for possible in (item, item.parent()):
            if hasattr(possible,"customdata") and hasattr(possible.customdata,"pageid"):
                    pageid=possible.customdata.pageid
                    break

        if pageid is not None:
            self.mainAnalysisTabRequestResponse.fill(str(pageid))
    
    ####
    # Requester tool section
    ####

    def main_tab_change(self):
        """ This function fires when the main tab widget changes. """
        
        position = self.mainTabWidget.currentIndex()
        
        if self.cookiesTabIndex == position:
            self.cookiesTab.fill_cookies_tab()
    
    ####
    # Web Fuzzer tool section
    ####
        
    def check_content_type(self, content):
        """ Check the content type of submitted content """
        
        # TODO: improve these to match tighter
        pattern_xml = re.compile("xml", re.I)
        pattern_js = re.compile("javascript", re.I)
        pattern_json = re.compile("json", re.I)
        pattern_css = re.compile("css", re.I)
        
        # Return the lexer type
        if pattern_xml.search(content):
            return("xml")
        elif pattern_js.search(content):
            return("javascript")
        elif pattern_json.search(content):
            return("javascript")
        elif pattern_css.search(content):
            return("css")
        else:
            return(None)

    def zoom_in(self):
        """ Zoom in on the items in the selected tab """
        self.framework.signal_zoom_in()

    def zoom_out(self):
        """ Zoom out on the items in the selected tab """
        self.framework.signal_zoom_out()

    def sitemap_context_menu(self, point):
        """ Display the context menu for the sitemap """
        self.treeViewSitemapMenu.exec_(self.treeViewSitemap.mapToGlobal(point))

    def sitemap_copy_url(self):
        index = self.treeViewSitemap.currentIndex()
        if index and index.isValid():
            obj = index.internalPointer()
            if obj.url:
                QApplication.clipboard().setText(obj.url)

    def diff(self):
        """ Launch Diff dialog and Diff 2 responses """
        
        myDiffDialog = DiffDialog(self.framework)
        myDiffDialog.show()
        myDiffDialog.exec_()

    def raft_config_toggle(self, configName, status):
        configValue = 'False'
        if status:
            configValue = 'True'
        self.framework.set_raft_config_value(configName, configValue)

    def display_about(self):
        dialog = RaftAboutDialog()
        dialog.show()
        dialog.exec_()
        
    def display_message(self, message):
        dialog = SimpleDialog(message)
        dialog.exec_()

    def display_confirm_dialog(self, message):
        response = QMessageBox.question(self, 'Confirm', message, QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
        if 0 != (response & QMessageBox.Yes):
            return True
        else:
            return False
    
    def detach_encoder(self):
        index = self.mainTabWidget.indexOf(self.tabMainEncoder)
        if -1 != index:
            tabText = self.mainTabWidget.tabText(index)
            dialog = QDialog()
            dialog.setWindowTitle(tabText)
            verticalLayout = QVBoxLayout(dialog)
            tabWidget = QTabWidget(dialog)
            verticalLayout.addWidget(tabWidget)
            self.tabMainEncoder.setParent(tabWidget)
            tabWidget.addTab(self.tabMainEncoder, tabText)
            dialog.finished.connect(lambda code: self.reattach_encoder(code, index))
            dialog.show()
            dialog.exec_()

    def reattach_encoder(self, code, index):
        self.tabMainEncoder.setParent(self.mainTabWidget)
        self.mainTabWidget.insertTab(index, self.tabMainEncoder, 'Encoder')
        
    def display_config(self):
        dialog = ConfigDialog(self.framework, self)
        dialog.show()
        dialog.exec_()
        
    def display_analysis_config(self):
        dialog = AnalysisConfigDialog(self.framework, self)
        
        #Instantiate all found analyzers
        self.analyzerlist.instantiate_analyzers(useallanalyzers=True)
        analyzerdict=TreeWidgetTools.obj_list_to_dict(self.analyzerlist, valueattr='isenabled')
        
        TreeWidgetTools.populate_tree_widget(dialog.analyzerList,analyzerdict)
        dialog.analyzerList.setSortingEnabled(True)
        dialog.analyzerList.sortItems(0,0)
        #generated.setParent(dialog.LeftWidget)
        #dialog.verticalLayoutTopLeft.addWidget(generated)
        
        #dialog.analyzerList=generated
        
        dialog.analyzerList.clicked.connect(dialog.viewItemSelected)
        dialog.defaultsButton.clicked.connect(dialog.defaultsButtonClicked)
        dialog.saveAllButton.clicked.connect(dialog.saveAllButtonClicked)
        dialog.closeButton.clicked.connect(dialog.closeButtonClicked)
        dialog.saveButton.clicked.connect(dialog.saveButtonClicked)
        dialog.addnodeButton.clicked.connect(dialog.addnodeButtonClicked)
        dialog.delnodeButton.clicked.connect(dialog.delnodeButtonClicked)
        
        #Setup complete, display dialog
        dialog.show()
        dialog.exec_()
        
    def display_sequence(self):
        dialog = SequenceDialog(self.framework)
        dialog.show()
        dialog.exec_()

    def display_search(self):
        dialog = SearchDialog(self.framework, self)
        dialog.show()
        dialog.exec_()

    def launch_browser(self):
        dialog = RaftBrowserDialog(self.framework, self)
        dialog.show()
        dialog.exec_()

    def open_url_in_browser(self, url):
        options = {'url':url}
        dialog = RaftBrowserDialog(self.framework, self, options)
        dialog.show()
        dialog.exec_()

    def open_content_in_browser(self, url, body, mimetype = ''):
        options = {'url':url, 'body':body, 'mimetype':mimetype}
        dialog = RaftBrowserDialog(self.framework, self, options)
        dialog.show()
        dialog.exec_()
        
    def test(self):
        """ Test Function """
        print("hello world")
示例#12
0
class RequesterTab(QObject):
    def __init__(self, framework, mainWindow):
        QObject.__init__(self, mainWindow)
        self.framework = framework
        self.mainWindow = mainWindow

        self.mainWindow.requesterSendButton.clicked.connect(self.requester_send_button_clicked)
        self.mainWindow.bulkRequestPushButton.clicked.connect(self.requester_bulk_request_button_clicked)
        self.mainWindow.requesterHistoryClearButton.clicked.connect(self.requester_history_clear_button_clicked)
        self.mainWindow.reqTabWidget.currentChanged.connect(self.handle_tab_currentChanged)
        self.mainWindow.requesterSequenceCheckBox.stateChanged.connect(self.handle_requesterSequenceCheckBox_stateChanged)
        self.mainWindow.bulkRequestSequenceCheckBox.stateChanged.connect(self.handle_bulkRequestSequenceCheckBox_stateChanged)
        self.mainWindow.sequenceRunnerRunButton.clicked.connect(self.handle_sequenceRunnerRunButton_clicked)
        self.pending_request = None
        self.pending_bulk_requests = None
        self.pending_sequence_requests = None

        self.re_request = re.compile(r'^(\S+)\s+((?:https?://(?:\S+\.)+\w+(?::\d+)?)?/.*)\s+HTTP/\d+\.\d+\s*$', re.I)
        self.re_request_cookie = re.compile(r'^Cookie:\s*(\S+)', re.I|re.M)
        self.re_replacement = re.compile(r'\$\{(\w+)\}')

        self.framework.subscribe_populate_requester_response_id(self.requester_populate_response_id)
        self.framework.subscribe_populate_bulk_requester_responses(self.bulk_requester_populate_responses)
        self.framework.subscribe_sequences_changed(self.fill_sequences)

        self.setup_requester_tab()

        self.Data = None
        self.cursor = None
        self.framework.subscribe_database_events(self.db_attach, self.db_detach)

    def db_attach(self):
        self.Data = self.framework.getDB()
        self.cursor = self.Data.allocate_thread_cursor()
        self.fill_requesters()

    def db_detach(self):
        self.close_cursor()
        self.Data = None

    def close_cursor(self):
        if self.cursor and self.Data:
            self.cursor.close()
            self.Data.release_thread_cursor(self.cursor)
            self.cursor = None

    def setup_requester_tab(self):

        self.historyRequestResponse = RequestResponseWidget(self.framework, self.mainWindow.requesterHistoryTabWidget, self.mainWindow.requesterHistorySearchResultsPlaceholder, self)
        self.requesterHistoryDataModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
        self.mainWindow.requesterHistoryTreeView.setModel(self.requesterHistoryDataModel)
        self.mainWindow.requesterHistoryTreeView.activated.connect(self.fill_history_request_response)
        self.mainWindow.requesterHistoryTreeView.clicked.connect(self.fill_history_request_response)
        self.mainWindow.requesterHistoryTreeView.doubleClicked.connect(self.requester_history_item_double_clicked)
        self.historyResponsesContextMenu = ResponsesContextMenuWidget(self.framework, self.requesterHistoryDataModel, self.mainWindow.requesterHistoryTreeView, self)
        self.historyResponsesContextMenu.set_currentChanged_callback(self.fill_history_request_response)

        self.sequenceRunnerRequestResponse = RequestResponseWidget(self.framework, self.mainWindow.sequenceRunnerTabWidget, self.mainWindow.sequenceRunnerSearchResultsPlaceholder, self)
        self.sequenceRunnerDataModel = ResponsesDataModel.ResponsesDataModel(self.framework, self)
        self.mainWindow.sequenceRunnerTreeView.setModel(self.sequenceRunnerDataModel)
        self.mainWindow.sequenceRunnerTreeView.activated.connect(self.fill_sequence_runner_request_response)
        self.mainWindow.sequenceRunnerTreeView.clicked.connect(self.fill_sequence_runner_request_response)
        self.mainWindow.sequenceRunnerTreeView.doubleClicked.connect(self.requester_sequence_runner_item_double_clicked)
        self.sequence_runnerResponsesContextMenu = ResponsesContextMenuWidget(self.framework, self.sequenceRunnerDataModel, self.mainWindow.sequenceRunnerTreeView, self)
        self.sequence_runnerResponsesContextMenu.set_currentChanged_callback(self.fill_sequence_runner_request_response)

        self.miniResponseRenderWidget = MiniResponseRenderWidget(self.framework, self.mainWindow.reqRespTabWidget, True, self)

        self.scopeController = self.framework.getScopeController()

    def requester_history_item_double_clicked(self, index):
        Id = interface.index_to_id(self.requesterHistoryDataModel, index)
        if Id:
            dialog = RequestResponseDetailDialog(self.framework, Id, self.mainWindow)
            dialog.show()
            dialog.exec_()

    def fill_history_request_response(self, index):
        Id = interface.index_to_id(self.requesterHistoryDataModel, index)
        if Id:
            self.historyRequestResponse.fill(Id)

    def requester_sequence_runner_item_double_clicked(self, index):
        Id = interface.index_to_id(self.sequenceRunnerDataModel, index)
        if Id:
            dialog = RequestResponseDetailDialog(self.framework, Id, self.mainWindow)
            dialog.show()
            dialog.exec_()

    def fill_sequence_runner_request_response(self, index):
        Id = interface.index_to_id(self.sequenceRunnerDataModel, index)
        if Id:
            self.sequenceRunnerRequestResponse.fill(Id)

    def fill_requesters(self):
        # requesters
        self.requesterHistoryDataModel.clearModel()
        history_items = []
        for row in self.Data.get_all_requester_history(self.cursor):
            response_item = interface.data_row_to_response_items(row)
            history_items.append(response_item)
        self.requesterHistoryDataModel.append_data(history_items)

        self.fill_sequences()

        self.mainWindow.requesterUrlEdit.setText(self.framework.get_raft_config_value('requesterUrlEdit'))
        self.mainWindow.bulkRequestUrlListEdit.setPlainText(self.framework.get_raft_config_value('bulkRequestUrlListEdit'))

    def fill_sequences(self):
        self.fill_sequences_combo_box(self.mainWindow.requesterSequenceComboBox)
        self.fill_sequences_combo_box(self.mainWindow.bulkRequestSequenceComboBox)
        self.fill_sequences_combo_box(self.mainWindow.sequenceRunnerSequenceComboBox)

    def requester_populate_response_id(self, Id):
        row = self.Data.read_responses_by_id(self.cursor, Id)
        if not row:
            return

        responseItems = interface.data_row_to_response_items(row)

        method, url, template_text = self.generate_template_for_response_item(responseItems)

        self.set_combo_box_text(self.mainWindow.requesterRequestMethod, method.upper())
        self.mainWindow.requesterUrlEdit.setText(url)
        self.mainWindow.requesterTemplateEdit.setPlainText(template_text)

    def bulk_requester_populate_responses(self, id_list):

        url_list = []
        first = True
        for Id in id_list:
            row = self.Data.read_responses_by_id(self.cursor, Id)
            if not row:
                continue

            responseItems = interface.data_row_to_response_items(row)
            url = responseItems[ResponsesTable.URL]
            if url not in url_list:
                url_list.append(url)

            if first:
                method, url, template_text = self.generate_template_for_response_item(responseItems)
                self.set_combo_box_text(self.mainWindow.bulkRequestMethodEdit, method.upper())
                self.mainWindow.bulkRequestTemplateEdit.setPlainText(template_text)
                first = False

        self.mainWindow.bulkRequestUrlListEdit.setPlainText('\n'.join(url_list))

    def generate_template_for_response_item(self, responseItems):
        url = responseItems[ResponsesTable.URL]
        reqHeaders = str(responseItems[ResponsesTable.REQ_HEADERS], 'utf-8', 'ignore')
        reqData = str(responseItems[ResponsesTable.REQ_DATA], 'utf-8', 'ignore')
        method = responseItems[ResponsesTable.REQ_METHOD]
        splitted = urlparse.urlsplit(url)

        useragent = self.framework.useragent()
        has_cookie = False
        template = StringIO()
        template.write('${method} ${request_uri} HTTP/1.1\n')
        first = True
        for line in reqHeaders.splitlines():
            if not line:
                break
            if first and self.re_request.match(line):
                first = False
                continue
            if ':' in line:
                name, value = [v.strip() for v in line.split(':', 1)]
                lname = name.lower()
                if 'host' == lname:
                    if splitted.hostname and value == splitted.hostname:
                        template.write('Host: ${host}\n')
                        continue
                elif 'user-agent' == lname:
                    if useragent == value:
                        template.write('User-Agent: ${user_agent}\n')
                        continue
            template.write(line)
            template.write('\n')
        template.write('\n')
        template.write(reqData)

        return method, url, template.getvalue()

    def set_combo_box_text(self, comboBox, selectedText):
        index = comboBox.findText(selectedText)
        if -1 == index:
            comboBox.addItem(selectedText)
            index = comboBox.findText(selectedText)
        comboBox.setCurrentIndex(index)

    def handle_requesterSequenceCheckBox_stateChanged(self, state):
        self.mainWindow.requesterSequenceComboBox.setEnabled(self.mainWindow.requesterSequenceCheckBox.isChecked())

    def handle_bulkRequestSequenceCheckBox_stateChanged(self, state):
        self.mainWindow.bulkRequestSequenceComboBox.setEnabled(self.mainWindow.bulkRequestSequenceCheckBox.isChecked())

    def handle_tab_currentChanged(self, index):
        # TODO: must this hard-coded ?
        if 0 == index:
            self.fill_sequences_combo_box(self.mainWindow.requesterSequenceComboBox)
        elif 1 == index:
            self.fill_sequences_combo_box(self.mainWindow.bulkRequestSequenceComboBox)
        elif 2 == index:
            self.fill_sequences_combo_box(self.mainWindow.sequenceRunnerSequenceComboBox)

    def requester_send_button_clicked(self):
        """ Make a request from the Request tab """

        if 'Cancel' == self.mainWindow.requesterSendButton.text() and self.pending_request is not None:
            self.pending_request.cancel()
            self.pending_request = None
            self.mainWindow.requesterSendButton.setText('Send')
            return

        qurl = QUrl.fromUserInput(self.mainWindow.requesterUrlEdit.text())
        url = qurl.toEncoded().data().decode('utf-8')
        self.mainWindow.requesterUrlEdit.setText(url)

        self.framework.set_raft_config_value('requesterUrlEdit', url)
        templateText = str(self.mainWindow.requesterTemplateEdit.toPlainText())
        method = str(self.mainWindow.requesterRequestMethod.currentText())

        use_global_cookie_jar = self.mainWindow.requesterUseGlobalCookieJar.isChecked()
        replacements = self.build_replacements(method, url)
        (method, url, headers, body) = self.process_template(url, templateText, replacements)

        sequenceId = None
        if self.mainWindow.requesterSequenceCheckBox.isChecked():
            sequenceId = str(self.mainWindow.requesterSequenceComboBox.itemData(self.mainWindow.requesterSequenceComboBox.currentIndex()))
        self.requestRunner = RequestRunner(self.framework, self)
        if use_global_cookie_jar:
            self.requesterCookieJar = self.framework.get_global_cookie_jar()
        else:
            self.requesterCookieJar = InMemoryCookieJar(self.framework, self)
            
        self.requestRunner.setup(self.requester_response_received, self.requesterCookieJar, sequenceId)

        self.pending_request = self.requestRunner.queue_request(method, url, headers, body)
        self.mainWindow.requesterSendButton.setText('Cancel')
        self.miniResponseRenderWidget.clear_response_render()

    def requester_response_received(self, response_id, context):
        if 0 != response_id:
            row = self.Data.read_responses_by_id(self.cursor, response_id)
            if row:
                response_item = interface.data_row_to_response_items(row)
                self.Data.insert_requester_history(self.cursor, response_id)
                self.requesterHistoryDataModel.append_data([response_item])

                url = response_item[ResponsesTable.URL]
                req_headers = response_item[ResponsesTable.REQ_HEADERS]
                req_body = response_item[ResponsesTable.REQ_DATA]
                res_headers = response_item[ResponsesTable.RES_HEADERS]
                res_body = response_item[ResponsesTable.RES_DATA]
                res_content_type = response_item[ResponsesTable.RES_CONTENT_TYPE]

                self.miniResponseRenderWidget.populate_response_content(url, req_headers, req_body, res_headers, res_body, res_content_type)

        self.mainWindow.requesterSendButton.setText('Send')
        self.pending_request = None

    def requester_bulk_request_button_clicked(self):
        if 'Cancel' == self.mainWindow.bulkRequestPushButton.text() and self.pending_bulk_requests is not None:
            self.cancel_bulk_requests = True
            for context, pending_request in self.pending_bulk_requests.items():
                pending_request.cancel()
            self.pending_bulk_requests = None
            self.mainWindow.bulkRequestPushButton.setText('Send')
            self.mainWindow.bulkRequestProgressBar.setValue(0)
            return

        if self.pending_bulk_requests is None:
            self.pending_bulk_requests = {}

        method = str(self.mainWindow.bulkRequestMethodEdit.currentText())
        templateText = str(self.mainWindow.bulkRequestTemplateEdit.toPlainText())

        template_url = str(self.mainWindow.bulkRequestUrlEdit.text())

        url_list = str(self.mainWindow.bulkRequestUrlListEdit.toPlainText())
        self.framework.set_raft_config_value('bulkRequestUrlListEdit', url_list)
        request_urls = url_list.splitlines()
        self.mainWindow.bulkRequestProgressBar.setValue(0)
        self.mainWindow.bulkRequestProgressBar.setMaximum(len(request_urls))

        sequenceId = None
        if self.mainWindow.bulkRequestSequenceCheckBox.isChecked():
            sequenceId = str(self.mainWindow.bulkRequestSequenceComboBox.itemData(self.mainWindow.bulkRequestSequenceComboBox.currentIndex()))

        first = True
        self.cancel_bulk_requests = False
        for request_url in request_urls:
            if self.cancel_bulk_requests:
                break
            request_url = request_url.strip()
            if request_url:
                context = uuid.uuid4().hex
                # TODO: move this hack 
                if '$' in template_url:
                    replacements = self.build_replacements(method, request_url)
                    url = self.re_replacement.sub(lambda m: replacements.get(m.group(1)), template_url)
                else:
                    url = request_url

                if not self.scopeController.isUrlInScope(url, url):
                    self.framework.log_warning('skipping out of scope URL: %s' % (url))
                    self.mainWindow.bulkRequestProgressBar.setValue(self.mainWindow.bulkRequestProgressBar.value()+1)
                    continue
                
                use_global_cookie_jar = self.mainWindow.bulkRequestUseGlobalCookieJar.isChecked()
                replacements = self.build_replacements(method, url)
                (method, url, headers, body) = self.process_template(url, templateText, replacements)

                if first:
                    self.mainWindow.bulkRequestPushButton.setText('Cancel')
                    if use_global_cookie_jar:
                        self.bulkRequesterCookieJar = self.framework.get_global_cookie_jar()
                    else:
                        self.bulkRequesterCookieJar = InMemoryCookieJar(self.framework, self)
                    self.bulk_requestRunner = RequestRunner(self.framework, self)
                    self.bulk_requestRunner.setup(self.requester_bulk_response_received, self.bulkRequesterCookieJar, sequenceId)
                    first = False

                self.pending_bulk_requests[context] = self.bulk_requestRunner.queue_request(method, url, headers, body, context)

    def requester_bulk_response_received(self, response_id, context):
        self.mainWindow.bulkRequestProgressBar.setValue(self.mainWindow.bulkRequestProgressBar.value()+1)
        context = str(context)
        if self.pending_bulk_requests is not None:
            try:
                self.pending_bulk_requests.pop(context)
            except KeyError as e:
                pass
        if 0 != response_id:
            row = self.Data.read_responses_by_id(self.cursor, response_id)
            if row:
                response_item = interface.data_row_to_response_items(row)
                self.Data.insert_requester_history(self.cursor, response_id)
                self.requesterHistoryDataModel.append_data([response_item])

        finished = False
        if self.pending_bulk_requests is None or len(self.pending_bulk_requests) == 0:
            self.mainWindow.bulkRequestProgressBar.setValue(self.mainWindow.bulkRequestProgressBar.maximum())
            finished = True
        elif self.mainWindow.bulkRequestProgressBar.value() == self.mainWindow.bulkRequestProgressBar.maximum():
            finished = True
        if finished:
            self.mainWindow.bulkRequestPushButton.setText('Send')

    def handle_sequenceRunnerRunButton_clicked(self):
        """ Run a sequence """
        if 'Cancel' == self.mainWindow.sequenceRunnerRunButton.text() and self.pending_sequence_requests is not None:
            self.cancel_sequence_requests = True
            for context, pending_request in self.pending_sequence_requests.items():
                pending_request.cancel()
            self.pending_sequence_requests = None
            self.mainWindow.sequenceRunnerButton.setText('Send')
            self.mainWindow.sequenceRunnerButton.setValue(0)
            return

        self.sequenceRunnerDataModel.clearModel()

        sequenceId = str(self.mainWindow.sequenceRunnerSequenceComboBox.itemData(self.mainWindow.sequenceRunnerSequenceComboBox.currentIndex()))
        use_global_cookie_jar = self.mainWindow.sequenceRunnerUseGlobalCookieJar.isChecked()
        if use_global_cookie_jar:
            self.sequenceRunnerCookieJar = self.framework.get_global_cookie_jar()
        else:
            self.sequenceRunnerCookieJar = InMemoryCookieJar(self.framework, self)

        self.sequence_requestRunner = RequestRunner(self.framework, self)
        self.sequence_requestRunner.setup(self.sequence_runner_response_received, self.sequenceRunnerCookieJar, sequenceId)
        self.pending_sequence_requests = self.sequence_requestRunner.run_sequence()
        self.mainWindow.sequenceRunnerRunButton.setText('Cancel')

    def sequence_runner_response_received(self, response_id, context):
        context = str(context)
        if self.pending_sequence_requests is not None:
            try:
                self.pending_sequence_requests.pop(context)
            except KeyError as e:
                print((e))
                pass

        if 0 != response_id:
            row = self.Data.read_responses_by_id(self.cursor, response_id)
            if row:
                response_item = interface.data_row_to_response_items(row)
                self.sequenceRunnerDataModel.append_data([response_item])

        if self.pending_sequence_requests is None or len(self.pending_sequence_requests) == 0:
            self.mainWindow.sequenceRunnerRunButton.setText('Send')

    def requester_history_clear_button_clicked(self):
        self.Data.clear_requester_history(self.cursor)
        self.requesterHistoryDataModel.clearModel()

    def fill_sequences_combo_box(self, comboBox):
        selectedText = comboBox.currentText()

        comboBox.clear()
        for row in self.Data.get_all_sequences(self.cursor):
            sequenceItem = [m or '' for m in row]
            name = str(sequenceItem[1])
            Id = str(sequenceItem[0])
            item = comboBox.addItem(name, Id)

        if selectedText:
            index = comboBox.findText(selectedText)
            if index != -1:
                comboBox.setCurrentIndex(index)

    def build_replacements(self, method, url):
        replacements = {}
        splitted = urlparse.urlsplit(url)
        replacements['method'] = method.upper()
        replacements['url'] = url
        replacements['scheme'] = splitted.scheme or ''
        replacements['netloc'] = splitted.netloc or ''
        replacements['host'] = splitted.hostname or ''
        replacements['path'] = splitted.path or '/'
        replacements['query'] = splitted.query or ''
        replacements['fragment'] = splitted.fragment or ''
        replacements['request_uri'] = urlparse.urlunsplit(('', '', replacements['path'], replacements['query'], ''))
        replacements['user_agent'] = self.framework.useragent()
        return replacements

    def process_template(self, url, template, replacements):

        method, uri = '' ,''
        headers, body = '', ''

        # TODO: this allows for missing entries -- is this good?
        func = lambda m: replacements.get(m.group(1))

        prev = 0
        while True:
            n = template.find('\n', prev)
            if -1 == n:
                break
            if n > 0 and '\r' == template[n-1]:
                line = template[prev:n-1]
            else:
                line = template[prev:n]

            if 0 == len(line):
                # end of headers
                headers = template[0:n+1]
                body = template[n+1:]
                break
            prev = n + 1

        if not headers:
            headers = template
            body = ''
            
        # TODO: could work from ordered dict to main order?
        headers_dict = {}
        first = True
        for line in headers.splitlines():
            if not line:
                break
            if '$' in line:
                line = self.re_replacement.sub(func, line)
            if first:
                m = self.re_request.match(line)
                if not m:
                    raise Exception('Invalid HTTP request: failed to match request line: %s' % (line))
                method = m.group(1)
                uri = m.group(2)
                first = False
                continue

            if ':' in line:
                name, value = [v.strip() for v in line.split(':', 1)]
                headers_dict[name] = value
        
        if '$' in body:
            body = self.re_replacement.sub(func, body)

        url = urlparse.urljoin(url, uri)

        return (method, url, headers_dict, body)