def subtract_parent_from_delta_page(parent_page, delta_page):
result = DeltaPage(delta_page.id,
delta_page.url,
delta_page.html,
cookiesjar=delta_page.cookiejar,
depth=delta_page.current_depth,
generator=delta_page.generator,
parent_id=delta_page.parent_id)
result.delta_depth = delta_page.delta_depth
for link in delta_page.links:
if link not in parent_page.links:
result.links.append(link)
for d_clickable in delta_page.clickables:
clickable_is_already_in_main = False
for m_clickable in parent_page.clickables:
if d_clickable == m_clickable:
clickable_is_already_in_main = True
break
if clickable_is_already_in_main == False:
result.clickables.append(d_clickable)
for d_form in delta_page.forms:
form_is_already_in_main = False
for m_form in parent_page.forms:
if two_forms_are_equal(d_form, m_form):
form_is_already_in_main = True
break
if form_is_already_in_main == False:
result.forms.append(d_form)
result.ajax_requests = delta_page.ajax_requests # They are just capturing the new one
return result
def subtract_parent_from_delta_page(parent_page, delta_page):
result = DeltaPage(delta_page.id, delta_page.url, delta_page.html, cookiesjar=delta_page.cookiejar, depth=delta_page.current_depth, generator=delta_page.generator, parent_id=delta_page.parent_id)
result.delta_depth = delta_page.delta_depth
for link in delta_page.links:
if link not in parent_page.links:
result.links.append(link)
for d_clickable in delta_page.clickables:
clickable_is_already_in_main = False
for m_clickable in parent_page.clickables:
if d_clickable == m_clickable:
clickable_is_already_in_main = True
break
if clickable_is_already_in_main == False:
result.clickables.append(d_clickable)
for d_form in delta_page.forms:
form_is_already_in_main = False
for m_form in parent_page.forms:
if two_forms_are_equal(d_form, m_form):
form_is_already_in_main = True
break
if form_is_already_in_main == False:
result.forms.append(d_form)
result.ajax_requests = delta_page.ajax_requests # They are just capturing the new one
return result
def _parse_delta_page_from_db(self, current_session, page):
clickables = self.get_all_clickables_to_page_id_from_db(current_session, page['web_page_id'])
forms = self.get_all_forms_to_page_id_from_db(current_session, page['web_page_id'])
generator = self.clickables.find_one({"_id": page['generator']})
generator = self._parse_clickable_from_db_to_model(generator)
generator_requests = []
for g in page['generator_requests']:
generator_requests.append(self.get_asyncrequest_to_id(current_session, g))
result = DeltaPage(page['web_page_id'], page['url'], page['html'], None, page['current_depth'], generator, page['parent_id'], page['delta_depth'])
links = []
for link in page['links']:
links.append(self._parse_link_from_db(link))
result.links = links
result.forms = forms
result.clickables = clickables
result.generator_requests = generator_requests
return result
def _parse_delta_page_from_db(self, current_session, page):
clickables = self.get_all_clickables_to_page_id_from_db(
current_session, page['web_page_id'])
forms = self.get_all_forms_to_page_id_from_db(current_session,
page['web_page_id'])
generator = self.clickables.find_one({"_id": page['generator']})
generator = self._parse_clickable_from_db_to_model(generator)
generator_requests = []
for g in page['generator_requests']:
generator_requests.append(
self.get_asyncrequest_to_id(current_session, g))
result = DeltaPage(page['web_page_id'], page['url'], page['html'],
None, page['current_depth'], generator,
page['parent_id'], page['delta_depth'])
links = []
for link in page['links']:
links.append(self._parse_link_from_db(link))
result.links = links
result.forms = forms
result.clickables = clickables
result.generator_requests = generator_requests
return result
def execute(self,
webpage,
timeout=5,
element_to_click=None,
xhr_options=XHRBehavior.ObserveXHR,
pre_clicks=[]):
logging.debug(
"EventExecutor test started on {}... with {} and preclicks {}".
format(
webpage.url, "None"
if element_to_click == None else element_to_click.toString(),
[x.toString() for x in pre_clicks]))
self._analyzing_finished = False
self._loading_complete = False
self.xhr_options = xhr_options
self.element_to_click = None
self.ajax_requests = []
self._new_url = None
self.timeming_events = None
self._capturing_ajax = False
self._new_clickables = []
self.element_to_click = element_to_click
self.popup = None
self.mainFrame().setHtml(webpage.html, QUrl(webpage.url))
target_tag = element_to_click.dom_address.split(
"/") if element_to_click != None else "xyz"
target_tag = target_tag[-1]
if target_tag in ['video']:
return EventResult.UnsupportedTag, None
cbData = {
"element_to_click": element_to_click,
"pre_clicks": pre_clicks,
"webpage": webpage,
"xhr_options": xhr_options,
"timeout": timeout,
"self": self,
}
t = 0.0
while (not self._loading_complete
and t < timeout): # Waiting for finish processing
logging.debug("Waiting to load")
self._wait(0.1)
t += 0.1
if not self._loading_complete:
logging.debug("Timeout occurs while initial page loading...")
return EventResult.ErrorWhileInitialLoading, None
# Prepare Page for clicking...
self._wait(0.1)
for click in pre_clicks:
pre_click_elem = None
logging.debug("Click on: " + click.toString())
if click.id != None and click.id != "":
pre_click_elem = self.search_element_with_id(click.id)
if click.html_class != None and pre_click_elem == None:
pre_click_elem = self.search_element_with_class(
click.html_class, click.dom_address)
if pre_click_elem == None:
pre_click_elem = self.search_element_without_id_and_class(
click.dom_address)
if pre_click_elem is None:
logging.debug("Preclicking element not found")
return EventResult.PreviousClickNotFound, None
if "javascript:" not in click.event:
js_code = click.event
if js_code[0:2] == "on":
js_code = js_code[
2:] # if event beginns with on, escape it
js_code = "Simulate." + js_code + "(this);"
pre_click_elem.evaluateJavaScript(
js_code) # Waiting for finish processing
else:
pre_click_elem.evaluateJavaScript(
click.event[len("javascript:"):])
self._wait(self.wait_for_event)
is_key_event = False
real_clickable = None
# Now execute the target event
if element_to_click != None:
if "javascript:" not in element_to_click.event:
self._url_changed = False
js_code = element_to_click.event
if js_code[0:2] == "on":
js_code = js_code[2:] # if event begins with on, escape it
if js_code in self.key_events:
is_key_event = True
random_char = random.choice(string.ascii_letters)
js_code = "Simulate." + js_code + "(this, '" + random_char + "');"
else:
js_code = "Simulate." + js_code + "(this);"
else:
js_code = element_to_click.event[len("javascript:"):]
self.mainFrame().evaluateJavaScript(
self._addEventListener
) # This time it is here, because I dont want to have the initial addings
real_clickable = None
if element_to_click.id != None and element_to_click.id != "":
real_clickable = self.search_element_with_id(
element_to_click.id)
if element_to_click.html_class != None and real_clickable == None:
real_clickable = self.search_element_with_class(
element_to_click.html_class, element_to_click.dom_address)
if real_clickable == None:
real_clickable = self.search_element_without_id_and_class(
element_to_click.dom_address)
if real_clickable is None:
logging.debug("Target Clickable not found")
self.triggerAfterClicksHandler(
"afterclicks", {
"data": cbData,
"errorcode": EventResult.TargetElementNotFound
})
return EventResult.TargetElementNotFound, None
self._capturing_ajax = True
real_clickable.evaluateJavaScript(js_code)
self._wait(0.5)
self._capturing_ajax = False
links, clickables = extract_links(self.mainFrame(), webpage.url)
forms = extract_forms(self.mainFrame())
elements_with_event_properties = property_helper(self.mainFrame())
self.mainFrame().evaluateJavaScript(self._property_obs_js)
self._wait(0.1)
html = self.mainFrame().toHtml()
url = self.mainFrame().url().toString()
if is_key_event:
generator = KeyClickable(element_to_click, random_char)
else:
generator = element_to_click
if self._url_changed and self._new_url.toString() != webpage.url:
delta_page = DeltaPage(-1,
self._new_url.toString(),
html=None,
generator=generator,
parent_id=webpage.id,
cookiesjar=webpage.cookiejar)
self._analyzing_finished = True
self.triggerAfterClicksHandler("afterclicks", {
"data": cbData,
"errorcode": EventResult.URLChanged
})
self.mainFrame().setHtml(None)
return EventResult.URLChanged, delta_page
elif self.popup is not None:
logging.debug("Event creates Popup with Url: {}".format(
self.popup.mainFrame().url().toString()))
popup_url = self.popup.mainFrame().url().toString()
delta_page = DeltaPage(-1,
popup_url,
html=None,
generator=generator,
parent_id=webpage.id)
self.popup = None
self._analyzing_finished = True
self.triggerAfterClicksHandler(
"afterclicks", {
"data": cbData,
"errorcode": EventResult.CreatesPopup
})
self.mainFrame().setHtml(None)
return EventResult.CreatesPopup, delta_page
else:
delta_page = DeltaPage(-1,
webpage.url,
html,
generator=generator,
parent_id=webpage.id,
cookiesjar=webpage.cookiejar)
delta_page.clickables = self._new_clickables # Set by add eventlistener code
delta_page.clickables.extend(clickables)
delta_page.clickables.extend(elements_with_event_properties)
delta_page.clickables = purge_dublicates(delta_page.clickables)
try:
delta_page.clickables.remove(
self.element_to_click) # remove the clickable self...
except ValueError:
pass
delta_page.links = links
delta_page.forms = forms
delta_page.ajax_requests = self.ajax_requests
self._analyzing_finished = True
self.triggerAfterClicksHandler("afterclicks", {
"data": cbData,
"errorcode": EventResult.Ok
})
self.mainFrame().setHtml(None)
return EventResult.Ok, delta_page
def execute(self, webpage, timeout=5, element_to_click=None, xhr_options=XHRBehavior.ObserveXHR, pre_clicks=[]):
logging.debug(
"EventExecutor test started on {}...".format(webpage.url) + " with " + element_to_click.toString())
self._analyzing_finished = False
self._loading_complete = False
self.xhr_options = xhr_options
self.element_to_click = None
self.ajax_requests = []
self._new_url = None
self.timeming_events = None
self._capturing_ajax = False
self._new_clickables = []
self.element_to_click = element_to_click
self.popup = None
self.mainFrame().setHtml(webpage.html, QUrl(webpage.url))
target_tag = element_to_click.dom_address.split("/")
target_tag = target_tag[-1]
if target_tag in ['video']:
return EventResult.UnsupportedTag, None
t = 0.0
while (not self._loading_complete and t < timeout ): # Waiting for finish processing
self._wait(0.1)
t += 0.1
if not self._loading_complete:
logging.debug("Timeout occurs while initial page loading...")
return EventResult.ErrorWhileInitialLoading, None
# Prepare Page for clicking...
self._wait(0.1)
for click in pre_clicks:
pre_click_elem = None
logging.debug("Click on: " + click.toString())
if click.id != None and click.id != "":
pre_click_elem = self.search_element_with_id(click.id)
if click.html_class != None and pre_click_elem == None:
pre_click_elem = self.search_element_with_class(click.html_class, click.dom_address)
if pre_click_elem == None:
pre_click_elem = self.search_element_without_id_and_class(click.dom_address)
if pre_click_elem is None:
logging.debug("Preclicking element not found")
return EventResult.PreviousClickNotFound, None
if "javascript:" not in click.event:
js_code = click.event
if js_code[0:2] == "on":
js_code = js_code[2:] # if event beginns with on, escape it
js_code = "Simulate." + js_code + "(this);"
pre_click_elem.evaluateJavaScript(js_code) # Waiting for finish processing
else:
pre_click_elem.evaluateJavaScript(click.event[len("javascript:"):])
self._wait(self.wait_for_event)
is_key_event = False
# Now execute the target event
if "javascript:" not in element_to_click.event:
self._url_changed = False
js_code = element_to_click.event
if js_code[0:2] == "on":
js_code = js_code[2:] # if event begins with on, escape it
if js_code in self.key_events:
is_key_event = True
random_char = random.choice(string.ascii_letters)
js_code = "Simulate." + js_code + "(this, '" + random_char + "');"
else:
js_code = "Simulate." + js_code + "(this);"
else:
js_code = element_to_click.event[len("javascript:"):]
self.mainFrame().evaluateJavaScript(
self._addEventListener) # This time it is here, because I dont want to have the initial addings
real_clickable = None
if element_to_click.id != None and element_to_click.id != "":
real_clickable = self.search_element_with_id(element_to_click.id)
if element_to_click.html_class != None and real_clickable == None:
real_clickable = self.search_element_with_class(element_to_click.html_class, element_to_click.dom_address)
if real_clickable == None:
real_clickable = self.search_element_without_id_and_class(element_to_click.dom_address)
if real_clickable is None:
logging.debug("Target Clickable not found")
return EventResult.TargetElementNotFound, None
self._capturing_ajax = True
real_clickable.evaluateJavaScript(js_code)
self._wait(0.5)
self._capturing_ajax = False
links, clickables = extract_links(self.mainFrame(), webpage.url)
forms = extract_forms(self.mainFrame())
elements_with_event_properties = property_helper(self.mainFrame())
self.mainFrame().evaluateJavaScript(self._property_obs_js)
self._wait(0.1)
html = self.mainFrame().toHtml()
url = self.mainFrame().url().toString()
if is_key_event:
generator = KeyClickable(element_to_click, random_char)
else:
generator = element_to_click
if self._url_changed and self._new_url.toString() != webpage.url:
delta_page = DeltaPage(-1, self._new_url.toString(), html=None, generator=generator, parent_id=webpage.id,
cookiesjar=webpage.cookiejar)
self._analyzing_finished = True
self.mainFrame().setHtml(None)
return EventResult.URLChanged, delta_page
elif self.popup is not None:
logging.debug("Event creates Popup with Url: {}".format(self.popup.mainFrame().url().toString()))
popup_url = self.popup.mainFrame().url().toString()
delta_page = DeltaPage(-1, popup_url, html=None, generator=generator, parent_id=webpage.id)
self.popup = None
self._analyzing_finished = True
self.mainFrame().setHtml(None)
return EventResult.CreatesPopup, delta_page
else:
delta_page = DeltaPage(-1, webpage.url, html, generator=generator, parent_id=webpage.id,
cookiesjar=webpage.cookiejar)
delta_page.clickables = self._new_clickables # Set by add eventlistener code
delta_page.clickables.extend(clickables)
delta_page.clickables.extend(elements_with_event_properties)
delta_page.clickables = purge_dublicates(delta_page.clickables)
try:
delta_page.clickables.remove(self.element_to_click) # remove the clickable self...
except ValueError:
pass
delta_page.links = links
delta_page.forms = forms
delta_page.ajax_requests = self.ajax_requests
self._analyzing_finished = True
self.mainFrame().setHtml(None)
return EventResult.Ok, delta_page
