Beispiel #1
0
    def analyze(self, url_to_request, timeout=10, current_depth=None, method="GET", data={}):
        try:
            url_to_request = url_to_request.toString()
        except AttributeError:
            url_to_request = url_to_request

        logging.debug("Start analyzing the url {}...".format(url_to_request))
        self._timing_requests = []
        self._new_clickables = []
        self._timeming_events = []
        self._current_timeming_event = None
        self._loading_complete = False
        self._analyzing_finished = False
        self.response_code = {}
        if method == "GET":
            self.mainFrame().load(QUrl(url_to_request))
        else:
            request = self.make_request(url_to_request)
            data = self.post_data_to_array(data)
            request.setRawHeader("Content-Type", QByteArray("application/x-www-form-urlencoded"))
            self.mainFrame().load(request, QNetworkAccessManager.PostOperation, data)
        t = 0
        while not self._loading_complete and t < timeout:  # Waiting for finish processing
            self._wait(self.wait_for_processing)
            t += self.wait_for_processing

        videos = self.mainFrame().findAllElements("video")
        if len(videos) > 0:
            logging.debug("{} videos found... removing them")
            for video in videos:
                video.removeFromDocument()

        overall_waiting_time = t
        buffer = 250
        while len(self._timeming_events) > 0 and overall_waiting_time < timeout:
            self._current_timeming_event = self._timeming_events.pop(0)  # Take the first event(ordered by needed time
            self._waiting_for = self._current_timeming_event["event_type"]  # Setting kind of event
            waiting_time_in_milliseconds = (
                self._current_timeming_event["time"] - overall_waiting_time
            )  # Taking waiting time and convert it from milliseconds to seconds
            waiting_time_in_milliseconds = (waiting_time_in_milliseconds + buffer) / 1000.0
            if waiting_time_in_milliseconds < 0.0:
                waiting_time_in_milliseconds = 0
            self._wait(waiting_time_in_milliseconds)  # Waiting for 100 millisecond before expected event
            overall_waiting_time += waiting_time_in_milliseconds
        if overall_waiting_time < 0.5:
            self._wait((0.5 - overall_waiting_time))

        # Just for debugging
        # f = open("text.txt", "w")
        # f.write(self.mainFrame().toHtml())
        # f.close()
        base_url = self.mainFrame().findFirstElement("base")
        if base_url is not None:
            base_url = base_url.attribute("href")

        links, clickables = extract_links(self.mainFrame(), url_to_request)
        forms = extract_forms(self.mainFrame())
        elements_with_event_properties = property_helper(self.mainFrame())
        self.mainFrame().evaluateJavaScript(self._property_obs_js)
        self._wait(0.1)

        self._analyzing_finished = True
        html_after_timeouts = self.mainFrame().toHtml()
        response_url = self.mainFrame().url().toString()

        self.mainFrame().setHtml(None)
        self._new_clickables.extend(clickables)
        self._new_clickables.extend(elements_with_event_properties)
        self._new_clickables = purge_dublicates(self._new_clickables)
        response_code = None
        try:
            response_code = self.response_code[url_to_request]
        except KeyError:
            response_code = 200
        if response_code is None:
            response_code = 200
        try:
            current_page = WebPage(self.parent().get_next_page_id(), response_url, html_after_timeouts)
        except AttributeError:  # Attacker don't need this function...
            current_page = WebPage(42, response_url, html_after_timeouts)
        current_page.timing_requests = self._timing_requests
        current_page.clickables = self._new_clickables
        current_page.links = links
        current_page.forms = forms
        if base_url is not None and base_url != "":
            current_page.base_url = base_url
        return response_code, current_page
    def analyze(self,
                url_to_request,
                timeout=10,
                current_depth=None,
                method="GET",
                data={}):
        try:
            url_to_request = url_to_request.toString()
        except AttributeError:
            url_to_request = url_to_request

        logging.debug("Start analyzing the url {}...".format(url_to_request))
        self._timing_requests = []
        self._new_clickables = []
        self._timeming_events = []
        self._current_timeming_event = None
        self._loading_complete = False
        self._analyzing_finished = False
        self.response_code = {}
        if method == "GET":
            self.mainFrame().load(QUrl(url_to_request))
        else:
            request = self.make_request(url_to_request)
            data = self.post_data_to_array(data)
            request.setRawHeader(
                'Content-Type',
                QByteArray('application/x-www-form-urlencoded'))
            self.mainFrame().load(request, QNetworkAccessManager.PostOperation,
                                  data)
        t = 0
        while (not self._loading_complete
               and t < timeout):  # Waiting for finish processing
            self._wait(self.wait_for_processing)
            t += self.wait_for_processing

        videos = self.mainFrame().findAllElements("video")
        if len(videos) > 0:
            logging.debug("{} videos found... removing them")
            for video in videos:
                video.removeFromDocument()

        overall_waiting_time = t
        buffer = 250
        while len(
                self._timeming_events) > 0 and overall_waiting_time < timeout:
            self._current_timeming_event = self._timeming_events.pop(
                0)  # Take the first event(ordered by needed time
            self._waiting_for = self._current_timeming_event[
                'event_type']  # Setting kind of event
            waiting_time_in_milliseconds = (
                self._current_timeming_event["time"] - overall_waiting_time
            )  # Taking waiting time and convert it from milliseconds to seconds
            waiting_time_in_milliseconds = (
                (waiting_time_in_milliseconds + buffer) / 1000.0)
            if waiting_time_in_milliseconds < 0.0:
                waiting_time_in_milliseconds = 0
            self._wait(waiting_time_in_milliseconds
                       )  # Waiting for 100 millisecond before expected event
            overall_waiting_time += waiting_time_in_milliseconds
        if overall_waiting_time < 0.5:
            self._wait((0.5 - overall_waiting_time))

        # Just for debugging
        #f = open("text.txt", "w", encoding="utf-8")
        #f.write(self.mainFrame().toHtml())
        #f.close()
        base_url = self.mainFrame().findFirstElement("base")
        if base_url is not None:
            base_url = base_url.attribute("href")

        links, clickables = extract_links(self.mainFrame(), url_to_request)
        forms = extract_forms(self.mainFrame())
        elements_with_event_properties = property_helper(self.mainFrame())
        self.mainFrame().evaluateJavaScript(self._property_obs_js)
        self._wait(0.1)

        self._analyzing_finished = True
        html_after_timeouts = self.mainFrame().toHtml()
        response_url = self.mainFrame().url().toString()

        self.mainFrame().setHtml(None)
        self._new_clickables.extend(clickables)
        self._new_clickables.extend(elements_with_event_properties)
        self._new_clickables = purge_dublicates(self._new_clickables)
        response_code = None
        try:
            response_code = self.response_code[url_to_request]
        except KeyError:
            response_code = 200
        if response_code is None:
            response_code = 200
        try:
            current_page = WebPage(self.parent().get_next_page_id(),
                                   response_url, html_after_timeouts)
        except AttributeError:  #Attacker don't need this function...
            current_page = WebPage(42, response_url, html_after_timeouts)
        current_page.timing_requests = self._timing_requests
        current_page.clickables = self._new_clickables
        current_page.links = links
        current_page.forms = forms
        if base_url is not None and base_url != "":
            current_page.base_url = base_url
        return response_code, current_page
Beispiel #3
0
    def execute(self, webpage, timeout=5, element_to_click=None, xhr_options=XHRBehavior.ObserveXHR, pre_clicks=[]):
        logging.debug(
            "EventExecutor test started on {}...".format(webpage.url) + " with " + element_to_click.toString())
        self._analyzing_finished = False
        self._loading_complete = False
        self.xhr_options = xhr_options
        self.element_to_click = None
        self.ajax_requests = []
        self._new_url = None
        self.timeming_events = None
        self._capturing_ajax = False
        self._new_clickables = []
        self.element_to_click = element_to_click
        self.popup = None
        self.mainFrame().setHtml(webpage.html, QUrl(webpage.url))
        target_tag = element_to_click.dom_address.split("/")
        target_tag = target_tag[-1]
        if target_tag in ['video']:
            return EventResult.UnsupportedTag, None

        t = 0.0
        while (not self._loading_complete and t < timeout ):  # Waiting for finish processing
            self._wait(0.1)
            t += 0.1
        if not self._loading_complete:
            logging.debug("Timeout occurs while initial page loading...")
            return EventResult.ErrorWhileInitialLoading, None
        # Prepare Page for clicking...
        self._wait(0.1)
        for click in pre_clicks:
            pre_click_elem = None
            logging.debug("Click on: " + click.toString())
            if click.id != None and click.id != "":
                pre_click_elem = self.search_element_with_id(click.id)
            if click.html_class != None and pre_click_elem == None:
                pre_click_elem = self.search_element_with_class(click.html_class, click.dom_address)
            if pre_click_elem == None:
                pre_click_elem = self.search_element_without_id_and_class(click.dom_address)

            if pre_click_elem is None:
                logging.debug("Preclicking element not found")
                return EventResult.PreviousClickNotFound, None

            if "javascript:" not in click.event:
                js_code = click.event
                if js_code[0:2] == "on":
                    js_code = js_code[2:]  # if event beginns with on, escape it
                js_code = "Simulate." + js_code + "(this);"
                pre_click_elem.evaluateJavaScript(js_code)  # Waiting for finish processing
            else:
                pre_click_elem.evaluateJavaScript(click.event[len("javascript:"):])
            self._wait(self.wait_for_event)

        is_key_event = False
            # Now execute the target event
        if "javascript:" not in element_to_click.event:
            self._url_changed = False
            js_code = element_to_click.event
            if js_code[0:2] == "on":
                js_code = js_code[2:]  # if event begins with on, escape it

            if js_code in self.key_events:
                is_key_event = True
                random_char = random.choice(string.ascii_letters)
                js_code = "Simulate." + js_code + "(this, '" + random_char + "');"
            else:
                js_code = "Simulate." + js_code + "(this);"
        else:
            js_code = element_to_click.event[len("javascript:"):]

        self.mainFrame().evaluateJavaScript(
            self._addEventListener)  # This time it is here, because I dont want to have the initial addings

        real_clickable = None
        if element_to_click.id != None and element_to_click.id != "":
            real_clickable = self.search_element_with_id(element_to_click.id)
        if element_to_click.html_class != None and real_clickable == None:
            real_clickable = self.search_element_with_class(element_to_click.html_class, element_to_click.dom_address)
        if real_clickable == None:
            real_clickable = self.search_element_without_id_and_class(element_to_click.dom_address)

        if real_clickable is None:
            logging.debug("Target Clickable not found")
            return EventResult.TargetElementNotFound, None

        self._capturing_ajax = True
        real_clickable.evaluateJavaScript(js_code)
        self._wait(0.5)
        self._capturing_ajax = False
        links, clickables = extract_links(self.mainFrame(), webpage.url)

        forms = extract_forms(self.mainFrame())
        elements_with_event_properties = property_helper(self.mainFrame())
        self.mainFrame().evaluateJavaScript(self._property_obs_js)
        self._wait(0.1)

        html = self.mainFrame().toHtml()
        url = self.mainFrame().url().toString()

        if is_key_event:
            generator = KeyClickable(element_to_click, random_char)
        else:
            generator = element_to_click
        if self._url_changed and self._new_url.toString() != webpage.url:
            delta_page = DeltaPage(-1, self._new_url.toString(), html=None, generator=generator, parent_id=webpage.id,
                                   cookiesjar=webpage.cookiejar)
            self._analyzing_finished = True
            self.mainFrame().setHtml(None)
            return EventResult.URLChanged, delta_page
        elif self.popup is not None:
            logging.debug("Event creates Popup with Url: {}".format(self.popup.mainFrame().url().toString()))
            popup_url = self.popup.mainFrame().url().toString()
            delta_page = DeltaPage(-1, popup_url, html=None, generator=generator, parent_id=webpage.id)
            self.popup = None
            self._analyzing_finished = True
            self.mainFrame().setHtml(None)
            return EventResult.CreatesPopup, delta_page
        else:
            delta_page = DeltaPage(-1, webpage.url, html, generator=generator, parent_id=webpage.id,
                                   cookiesjar=webpage.cookiejar)
            delta_page.clickables = self._new_clickables  # Set by add eventlistener code
            delta_page.clickables.extend(clickables)
            delta_page.clickables.extend(elements_with_event_properties)
            delta_page.clickables = purge_dublicates(delta_page.clickables)
            try:
                delta_page.clickables.remove(self.element_to_click) # remove the clickable self...
            except ValueError:
                pass
            delta_page.links = links
            delta_page.forms = forms
            delta_page.ajax_requests = self.ajax_requests
            self._analyzing_finished = True
            self.mainFrame().setHtml(None)
            return EventResult.Ok, delta_page
Beispiel #4
0
    def execute(self,
                webpage,
                timeout=5,
                element_to_click=None,
                xhr_options=XHRBehavior.ObserveXHR,
                pre_clicks=[]):
        logging.debug(
            "EventExecutor test started on {}... with {} and preclicks {}".
            format(
                webpage.url, "None"
                if element_to_click == None else element_to_click.toString(),
                [x.toString() for x in pre_clicks]))
        self._analyzing_finished = False
        self._loading_complete = False
        self.xhr_options = xhr_options
        self.element_to_click = None
        self.ajax_requests = []
        self._new_url = None
        self.timeming_events = None
        self._capturing_ajax = False
        self._new_clickables = []
        self.element_to_click = element_to_click
        self.popup = None
        self.mainFrame().setHtml(webpage.html, QUrl(webpage.url))
        target_tag = element_to_click.dom_address.split(
            "/") if element_to_click != None else "xyz"
        target_tag = target_tag[-1]
        if target_tag in ['video']:
            return EventResult.UnsupportedTag, None

        cbData = {
            "element_to_click": element_to_click,
            "pre_clicks": pre_clicks,
            "webpage": webpage,
            "xhr_options": xhr_options,
            "timeout": timeout,
            "self": self,
        }

        t = 0.0
        while (not self._loading_complete
               and t < timeout):  # Waiting for finish processing
            logging.debug("Waiting to load")
            self._wait(0.1)
            t += 0.1
        if not self._loading_complete:
            logging.debug("Timeout occurs while initial page loading...")
            return EventResult.ErrorWhileInitialLoading, None
        # Prepare Page for clicking...
        self._wait(0.1)
        for click in pre_clicks:
            pre_click_elem = None
            logging.debug("Click on: " + click.toString())
            if click.id != None and click.id != "":
                pre_click_elem = self.search_element_with_id(click.id)
            if click.html_class != None and pre_click_elem == None:
                pre_click_elem = self.search_element_with_class(
                    click.html_class, click.dom_address)
            if pre_click_elem == None:
                pre_click_elem = self.search_element_without_id_and_class(
                    click.dom_address)

            if pre_click_elem is None:
                logging.debug("Preclicking element not found")
                return EventResult.PreviousClickNotFound, None

            if "javascript:" not in click.event:
                js_code = click.event
                if js_code[0:2] == "on":
                    js_code = js_code[
                        2:]  # if event beginns with on, escape it
                js_code = "Simulate." + js_code + "(this);"
                pre_click_elem.evaluateJavaScript(
                    js_code)  # Waiting for finish processing
            else:
                pre_click_elem.evaluateJavaScript(
                    click.event[len("javascript:"):])
            self._wait(self.wait_for_event)

        is_key_event = False
        real_clickable = None
        # Now execute the target event
        if element_to_click != None:
            if "javascript:" not in element_to_click.event:
                self._url_changed = False
                js_code = element_to_click.event
                if js_code[0:2] == "on":
                    js_code = js_code[2:]  # if event begins with on, escape it

                if js_code in self.key_events:
                    is_key_event = True
                    random_char = random.choice(string.ascii_letters)
                    js_code = "Simulate." + js_code + "(this, '" + random_char + "');"
                else:
                    js_code = "Simulate." + js_code + "(this);"
            else:
                js_code = element_to_click.event[len("javascript:"):]

            self.mainFrame().evaluateJavaScript(
                self._addEventListener
            )  # This time it is here, because I dont want to have the initial addings

            real_clickable = None
            if element_to_click.id != None and element_to_click.id != "":
                real_clickable = self.search_element_with_id(
                    element_to_click.id)
            if element_to_click.html_class != None and real_clickable == None:
                real_clickable = self.search_element_with_class(
                    element_to_click.html_class, element_to_click.dom_address)
            if real_clickable == None:
                real_clickable = self.search_element_without_id_and_class(
                    element_to_click.dom_address)

        if real_clickable is None:
            logging.debug("Target Clickable not found")
            self.triggerAfterClicksHandler(
                "afterclicks", {
                    "data": cbData,
                    "errorcode": EventResult.TargetElementNotFound
                })
            return EventResult.TargetElementNotFound, None

        self._capturing_ajax = True
        real_clickable.evaluateJavaScript(js_code)
        self._wait(0.5)
        self._capturing_ajax = False
        links, clickables = extract_links(self.mainFrame(), webpage.url)

        forms = extract_forms(self.mainFrame())
        elements_with_event_properties = property_helper(self.mainFrame())
        self.mainFrame().evaluateJavaScript(self._property_obs_js)
        self._wait(0.1)

        html = self.mainFrame().toHtml()
        url = self.mainFrame().url().toString()

        if is_key_event:
            generator = KeyClickable(element_to_click, random_char)
        else:
            generator = element_to_click
        if self._url_changed and self._new_url.toString() != webpage.url:
            delta_page = DeltaPage(-1,
                                   self._new_url.toString(),
                                   html=None,
                                   generator=generator,
                                   parent_id=webpage.id,
                                   cookiesjar=webpage.cookiejar)
            self._analyzing_finished = True
            self.triggerAfterClicksHandler("afterclicks", {
                "data": cbData,
                "errorcode": EventResult.URLChanged
            })
            self.mainFrame().setHtml(None)
            return EventResult.URLChanged, delta_page
        elif self.popup is not None:
            logging.debug("Event creates Popup with Url: {}".format(
                self.popup.mainFrame().url().toString()))
            popup_url = self.popup.mainFrame().url().toString()
            delta_page = DeltaPage(-1,
                                   popup_url,
                                   html=None,
                                   generator=generator,
                                   parent_id=webpage.id)
            self.popup = None
            self._analyzing_finished = True
            self.triggerAfterClicksHandler(
                "afterclicks", {
                    "data": cbData,
                    "errorcode": EventResult.CreatesPopup
                })
            self.mainFrame().setHtml(None)
            return EventResult.CreatesPopup, delta_page
        else:
            delta_page = DeltaPage(-1,
                                   webpage.url,
                                   html,
                                   generator=generator,
                                   parent_id=webpage.id,
                                   cookiesjar=webpage.cookiejar)
            delta_page.clickables = self._new_clickables  # Set by add eventlistener code
            delta_page.clickables.extend(clickables)
            delta_page.clickables.extend(elements_with_event_properties)
            delta_page.clickables = purge_dublicates(delta_page.clickables)
            try:
                delta_page.clickables.remove(
                    self.element_to_click)  # remove the clickable self...
            except ValueError:
                pass
            delta_page.links = links
            delta_page.forms = forms
            delta_page.ajax_requests = self.ajax_requests
            self._analyzing_finished = True
            self.triggerAfterClicksHandler("afterclicks", {
                "data": cbData,
                "errorcode": EventResult.Ok
            })
            self.mainFrame().setHtml(None)
            return EventResult.Ok, delta_page
Beispiel #5
0
    def submit_form(self, form, webpage, data=dict(), timeout=5):
        logging.debug("FormHandler on Page: {} started...".format(webpage.url))
        self._loading_complete = False
        self._analyzing_finished = False
        try:
            url = webpage.url.toString()
        except AttributeError:
            url = webpage.url
        self.mainFrame().setHtml(webpage.html, QUrl(url))
        self._new_clickables = []

        t = 0.0
        while not self._loading_complete and t < timeout:  # Waiting for finish processing
            self._wait(0.1)
            t += 0.1
        if not self._loading_complete:
            logging.debug("Timeout occurs while initial page loading...")
            return EventResult.ErrorWhileInitialLoading, None

        target_form = None
        p_forms = self.mainFrame().findAllElements("form")
        for tmp_form in p_forms:
            path = tmp_form.evaluateJavaScript("getXPath(this)")
            if path == form.dom_address:
                target_form = tmp_form
                break
        if target_form is None:
            return EventResult.TargetElementNotFound, None

        for elem in form.parameter:  #Iterate through abstract form representation
            if elem.name in data:  #Check if we have the data we must set
                elem_found = False  # Indicates if we found the element in the html
                value_to_set = data[elem.name]
                for tmp in target_form.findAll(
                        elem.tag
                ):  #Locking in the target form, if we found the element we have to set
                    if tmp.attribute(
                            "name"
                    ) == elem.name:  # Has the current element in the html the same name as our data
                        tmp.evaluateJavaScript("this.value = '" +
                                               value_to_set + "';")
                        elem_found = True
                        break
                if not elem_found:
                    return EventResult.TargetElementNotFound, None
        # Now we should have set all known parameters, next click the submit button
        q_submit_button = None
        if "submit" in form.toString():
            inputs = target_form.findAll("input") + target_form.findAll(
                "button")
            for el in inputs:
                if el.attribute("type") == "submit":
                    q_submit_button = el
                    break
            #q_submit_button.evaluateJavaScript("this.id='oxyfrymbel'")
        else:
            logging.debug(form.toString())

        if q_submit_button is None:
            inputs = target_form.findAll("button")
            q_submit_button = None
            if len(inputs) > 1:
                logging.debug("Cannot locate login button...")
                return EventResult.TargetElementNotFound, None
            elif len(inputs) == 1:
                q_submit_button = inputs[0]

        method = target_form.attribute("onsubmit")
        if method is not None and method != "":
            js_code_snippets = method.split(";")
            for snippet in js_code_snippets:
                if "return" in snippet or snippet == "":
                    logging.debug("Ignoring snippet: {}".format(snippet))
                    continue
                logging.debug("Eval: {}".format(snippet + ";"))
                self.mainFrame().evaluateJavaScript(snippet + ";")
                self._wait(3)
            self.mainFrame().evaluateJavaScript(self._addEventListener)
            self._wait(3)
        else:
            #TODO: Implement way for sending forms without onsubmit-method
            # check between: target_form.evaluateJavaScript("Simulate or document.?form?.submit())
            # or submit_button click
            if q_submit_button is not None:
                logging.debug("Click on submit button...")
                q_submit_button.evaluateJavaScript("Simulate.click(this);")
                self._wait(3)
            else:
                logging.debug("Trigger submit event on form...")
                target_form.evaluateJavaScript("Simulate.submit(this);")
                self._wait(3)

        links, clickables = extract_links(self.mainFrame(), url)
        forms = extract_forms(self.mainFrame())
        html = self.mainFrame().toHtml()
        #f = open("html.txt", "w", encoding="utf-8")
        #f.write(html)
        #f.close()
        self.mainFrame().setHtml(None)
        self._new_clickables.extend(clickables)
        self._new_clickables = purge_dublicates(self._new_clickables)
        return EventResult.Ok, html, self._new_clickables, forms, links, []