def subtract_parent_from_delta_page(parent_page, delta_page): result = DeltaPage(delta_page.id, delta_page.url, delta_page.html, cookiesjar=delta_page.cookiejar, depth=delta_page.current_depth, generator=delta_page.generator, parent_id=delta_page.parent_id) result.delta_depth = delta_page.delta_depth for link in delta_page.links: if link not in parent_page.links: result.links.append(link) for d_clickable in delta_page.clickables: clickable_is_already_in_main = False for m_clickable in parent_page.clickables: if d_clickable == m_clickable: clickable_is_already_in_main = True break if clickable_is_already_in_main == False: result.clickables.append(d_clickable) for d_form in delta_page.forms: form_is_already_in_main = False for m_form in parent_page.forms: if two_forms_are_equal(d_form, m_form): form_is_already_in_main = True break if form_is_already_in_main == False: result.forms.append(d_form) result.ajax_requests = delta_page.ajax_requests # They are just capturing the new one return result
def _parse_delta_page_from_db(self, current_session, page): clickables = self.get_all_clickables_to_page_id_from_db(current_session, page['web_page_id']) forms = self.get_all_forms_to_page_id_from_db(current_session, page['web_page_id']) generator = self.clickables.find_one({"_id": page['generator']}) generator = self._parse_clickable_from_db_to_model(generator) generator_requests = [] for g in page['generator_requests']: generator_requests.append(self.get_asyncrequest_to_id(current_session, g)) result = DeltaPage(page['web_page_id'], page['url'], page['html'], None, page['current_depth'], generator, page['parent_id'], page['delta_depth']) links = [] for link in page['links']: links.append(self._parse_link_from_db(link)) result.links = links result.forms = forms result.clickables = clickables result.generator_requests = generator_requests return result
def _parse_delta_page_from_db(self, current_session, page): clickables = self.get_all_clickables_to_page_id_from_db( current_session, page['web_page_id']) forms = self.get_all_forms_to_page_id_from_db(current_session, page['web_page_id']) generator = self.clickables.find_one({"_id": page['generator']}) generator = self._parse_clickable_from_db_to_model(generator) generator_requests = [] for g in page['generator_requests']: generator_requests.append( self.get_asyncrequest_to_id(current_session, g)) result = DeltaPage(page['web_page_id'], page['url'], page['html'], None, page['current_depth'], generator, page['parent_id'], page['delta_depth']) links = [] for link in page['links']: links.append(self._parse_link_from_db(link)) result.links = links result.forms = forms result.clickables = clickables result.generator_requests = generator_requests return result
def execute(self, webpage, timeout=5, element_to_click=None, xhr_options=XHRBehavior.ObserveXHR, pre_clicks=[]): logging.debug( "EventExecutor test started on {}... with {} and preclicks {}". format( webpage.url, "None" if element_to_click == None else element_to_click.toString(), [x.toString() for x in pre_clicks])) self._analyzing_finished = False self._loading_complete = False self.xhr_options = xhr_options self.element_to_click = None self.ajax_requests = [] self._new_url = None self.timeming_events = None self._capturing_ajax = False self._new_clickables = [] self.element_to_click = element_to_click self.popup = None self.mainFrame().setHtml(webpage.html, QUrl(webpage.url)) target_tag = element_to_click.dom_address.split( "/") if element_to_click != None else "xyz" target_tag = target_tag[-1] if target_tag in ['video']: return EventResult.UnsupportedTag, None cbData = { "element_to_click": element_to_click, "pre_clicks": pre_clicks, "webpage": webpage, "xhr_options": xhr_options, "timeout": timeout, "self": self, } t = 0.0 while (not self._loading_complete and t < timeout): # Waiting for finish processing logging.debug("Waiting to load") self._wait(0.1) t += 0.1 if not self._loading_complete: logging.debug("Timeout occurs while initial page loading...") return EventResult.ErrorWhileInitialLoading, None # Prepare Page for clicking... self._wait(0.1) for click in pre_clicks: pre_click_elem = None logging.debug("Click on: " + click.toString()) if click.id != None and click.id != "": pre_click_elem = self.search_element_with_id(click.id) if click.html_class != None and pre_click_elem == None: pre_click_elem = self.search_element_with_class( click.html_class, click.dom_address) if pre_click_elem == None: pre_click_elem = self.search_element_without_id_and_class( click.dom_address) if pre_click_elem is None: logging.debug("Preclicking element not found") return EventResult.PreviousClickNotFound, None if "javascript:" not in click.event: js_code = click.event if js_code[0:2] == "on": js_code = js_code[ 2:] # if event beginns with on, escape it js_code = "Simulate." + js_code + "(this);" pre_click_elem.evaluateJavaScript( js_code) # Waiting for finish processing else: pre_click_elem.evaluateJavaScript( click.event[len("javascript:"):]) self._wait(self.wait_for_event) is_key_event = False real_clickable = None # Now execute the target event if element_to_click != None: if "javascript:" not in element_to_click.event: self._url_changed = False js_code = element_to_click.event if js_code[0:2] == "on": js_code = js_code[2:] # if event begins with on, escape it if js_code in self.key_events: is_key_event = True random_char = random.choice(string.ascii_letters) js_code = "Simulate." + js_code + "(this, '" + random_char + "');" else: js_code = "Simulate." + js_code + "(this);" else: js_code = element_to_click.event[len("javascript:"):] self.mainFrame().evaluateJavaScript( self._addEventListener ) # This time it is here, because I dont want to have the initial addings real_clickable = None if element_to_click.id != None and element_to_click.id != "": real_clickable = self.search_element_with_id( element_to_click.id) if element_to_click.html_class != None and real_clickable == None: real_clickable = self.search_element_with_class( element_to_click.html_class, element_to_click.dom_address) if real_clickable == None: real_clickable = self.search_element_without_id_and_class( element_to_click.dom_address) if real_clickable is None: logging.debug("Target Clickable not found") self.triggerAfterClicksHandler( "afterclicks", { "data": cbData, "errorcode": EventResult.TargetElementNotFound }) return EventResult.TargetElementNotFound, None self._capturing_ajax = True real_clickable.evaluateJavaScript(js_code) self._wait(0.5) self._capturing_ajax = False links, clickables = extract_links(self.mainFrame(), webpage.url) forms = extract_forms(self.mainFrame()) elements_with_event_properties = property_helper(self.mainFrame()) self.mainFrame().evaluateJavaScript(self._property_obs_js) self._wait(0.1) html = self.mainFrame().toHtml() url = self.mainFrame().url().toString() if is_key_event: generator = KeyClickable(element_to_click, random_char) else: generator = element_to_click if self._url_changed and self._new_url.toString() != webpage.url: delta_page = DeltaPage(-1, self._new_url.toString(), html=None, generator=generator, parent_id=webpage.id, cookiesjar=webpage.cookiejar) self._analyzing_finished = True self.triggerAfterClicksHandler("afterclicks", { "data": cbData, "errorcode": EventResult.URLChanged }) self.mainFrame().setHtml(None) return EventResult.URLChanged, delta_page elif self.popup is not None: logging.debug("Event creates Popup with Url: {}".format( self.popup.mainFrame().url().toString())) popup_url = self.popup.mainFrame().url().toString() delta_page = DeltaPage(-1, popup_url, html=None, generator=generator, parent_id=webpage.id) self.popup = None self._analyzing_finished = True self.triggerAfterClicksHandler( "afterclicks", { "data": cbData, "errorcode": EventResult.CreatesPopup }) self.mainFrame().setHtml(None) return EventResult.CreatesPopup, delta_page else: delta_page = DeltaPage(-1, webpage.url, html, generator=generator, parent_id=webpage.id, cookiesjar=webpage.cookiejar) delta_page.clickables = self._new_clickables # Set by add eventlistener code delta_page.clickables.extend(clickables) delta_page.clickables.extend(elements_with_event_properties) delta_page.clickables = purge_dublicates(delta_page.clickables) try: delta_page.clickables.remove( self.element_to_click) # remove the clickable self... except ValueError: pass delta_page.links = links delta_page.forms = forms delta_page.ajax_requests = self.ajax_requests self._analyzing_finished = True self.triggerAfterClicksHandler("afterclicks", { "data": cbData, "errorcode": EventResult.Ok }) self.mainFrame().setHtml(None) return EventResult.Ok, delta_page
def execute(self, webpage, timeout=5, element_to_click=None, xhr_options=XHRBehavior.ObserveXHR, pre_clicks=[]): logging.debug( "EventExecutor test started on {}...".format(webpage.url) + " with " + element_to_click.toString()) self._analyzing_finished = False self._loading_complete = False self.xhr_options = xhr_options self.element_to_click = None self.ajax_requests = [] self._new_url = None self.timeming_events = None self._capturing_ajax = False self._new_clickables = [] self.element_to_click = element_to_click self.popup = None self.mainFrame().setHtml(webpage.html, QUrl(webpage.url)) target_tag = element_to_click.dom_address.split("/") target_tag = target_tag[-1] if target_tag in ['video']: return EventResult.UnsupportedTag, None t = 0.0 while (not self._loading_complete and t < timeout ): # Waiting for finish processing self._wait(0.1) t += 0.1 if not self._loading_complete: logging.debug("Timeout occurs while initial page loading...") return EventResult.ErrorWhileInitialLoading, None # Prepare Page for clicking... self._wait(0.1) for click in pre_clicks: pre_click_elem = None logging.debug("Click on: " + click.toString()) if click.id != None and click.id != "": pre_click_elem = self.search_element_with_id(click.id) if click.html_class != None and pre_click_elem == None: pre_click_elem = self.search_element_with_class(click.html_class, click.dom_address) if pre_click_elem == None: pre_click_elem = self.search_element_without_id_and_class(click.dom_address) if pre_click_elem is None: logging.debug("Preclicking element not found") return EventResult.PreviousClickNotFound, None if "javascript:" not in click.event: js_code = click.event if js_code[0:2] == "on": js_code = js_code[2:] # if event beginns with on, escape it js_code = "Simulate." + js_code + "(this);" pre_click_elem.evaluateJavaScript(js_code) # Waiting for finish processing else: pre_click_elem.evaluateJavaScript(click.event[len("javascript:"):]) self._wait(self.wait_for_event) is_key_event = False # Now execute the target event if "javascript:" not in element_to_click.event: self._url_changed = False js_code = element_to_click.event if js_code[0:2] == "on": js_code = js_code[2:] # if event begins with on, escape it if js_code in self.key_events: is_key_event = True random_char = random.choice(string.ascii_letters) js_code = "Simulate." + js_code + "(this, '" + random_char + "');" else: js_code = "Simulate." + js_code + "(this);" else: js_code = element_to_click.event[len("javascript:"):] self.mainFrame().evaluateJavaScript( self._addEventListener) # This time it is here, because I dont want to have the initial addings real_clickable = None if element_to_click.id != None and element_to_click.id != "": real_clickable = self.search_element_with_id(element_to_click.id) if element_to_click.html_class != None and real_clickable == None: real_clickable = self.search_element_with_class(element_to_click.html_class, element_to_click.dom_address) if real_clickable == None: real_clickable = self.search_element_without_id_and_class(element_to_click.dom_address) if real_clickable is None: logging.debug("Target Clickable not found") return EventResult.TargetElementNotFound, None self._capturing_ajax = True real_clickable.evaluateJavaScript(js_code) self._wait(0.5) self._capturing_ajax = False links, clickables = extract_links(self.mainFrame(), webpage.url) forms = extract_forms(self.mainFrame()) elements_with_event_properties = property_helper(self.mainFrame()) self.mainFrame().evaluateJavaScript(self._property_obs_js) self._wait(0.1) html = self.mainFrame().toHtml() url = self.mainFrame().url().toString() if is_key_event: generator = KeyClickable(element_to_click, random_char) else: generator = element_to_click if self._url_changed and self._new_url.toString() != webpage.url: delta_page = DeltaPage(-1, self._new_url.toString(), html=None, generator=generator, parent_id=webpage.id, cookiesjar=webpage.cookiejar) self._analyzing_finished = True self.mainFrame().setHtml(None) return EventResult.URLChanged, delta_page elif self.popup is not None: logging.debug("Event creates Popup with Url: {}".format(self.popup.mainFrame().url().toString())) popup_url = self.popup.mainFrame().url().toString() delta_page = DeltaPage(-1, popup_url, html=None, generator=generator, parent_id=webpage.id) self.popup = None self._analyzing_finished = True self.mainFrame().setHtml(None) return EventResult.CreatesPopup, delta_page else: delta_page = DeltaPage(-1, webpage.url, html, generator=generator, parent_id=webpage.id, cookiesjar=webpage.cookiejar) delta_page.clickables = self._new_clickables # Set by add eventlistener code delta_page.clickables.extend(clickables) delta_page.clickables.extend(elements_with_event_properties) delta_page.clickables = purge_dublicates(delta_page.clickables) try: delta_page.clickables.remove(self.element_to_click) # remove the clickable self... except ValueError: pass delta_page.links = links delta_page.forms = forms delta_page.ajax_requests = self.ajax_requests self._analyzing_finished = True self.mainFrame().setHtml(None) return EventResult.Ok, delta_page