def analyze(self, url_to_request, timeout=10, current_depth=None, method="GET", data={}):
try:
url_to_request = url_to_request.toString()
except AttributeError:
url_to_request = url_to_request
logging.debug("Start analyzing the url {}...".format(url_to_request))
self._timing_requests = []
self._new_clickables = []
self._timeming_events = []
self._current_timeming_event = None
self._loading_complete = False
self._analyzing_finished = False
self.response_code = {}
if method == "GET":
self.mainFrame().load(QUrl(url_to_request))
else:
request = self.make_request(url_to_request)
data = self.post_data_to_array(data)
request.setRawHeader("Content-Type", QByteArray("application/x-www-form-urlencoded"))
self.mainFrame().load(request, QNetworkAccessManager.PostOperation, data)
t = 0
while not self._loading_complete and t < timeout: # Waiting for finish processing
self._wait(self.wait_for_processing)
t += self.wait_for_processing
videos = self.mainFrame().findAllElements("video")
if len(videos) > 0:
logging.debug("{} videos found... removing them")
for video in videos:
video.removeFromDocument()
overall_waiting_time = t
buffer = 250
while len(self._timeming_events) > 0 and overall_waiting_time < timeout:
self._current_timeming_event = self._timeming_events.pop(0) # Take the first event(ordered by needed time
self._waiting_for = self._current_timeming_event["event_type"] # Setting kind of event
waiting_time_in_milliseconds = (
self._current_timeming_event["time"] - overall_waiting_time
) # Taking waiting time and convert it from milliseconds to seconds
waiting_time_in_milliseconds = (waiting_time_in_milliseconds + buffer) / 1000.0
if waiting_time_in_milliseconds < 0.0:
waiting_time_in_milliseconds = 0
self._wait(waiting_time_in_milliseconds) # Waiting for 100 millisecond before expected event
overall_waiting_time += waiting_time_in_milliseconds
if overall_waiting_time < 0.5:
self._wait((0.5 - overall_waiting_time))
# Just for debugging
# f = open("text.txt", "w")
# f.write(self.mainFrame().toHtml())
# f.close()
base_url = self.mainFrame().findFirstElement("base")
if base_url is not None:
base_url = base_url.attribute("href")
links, clickables = extract_links(self.mainFrame(), url_to_request)
forms = extract_forms(self.mainFrame())
elements_with_event_properties = property_helper(self.mainFrame())
self.mainFrame().evaluateJavaScript(self._property_obs_js)
self._wait(0.1)
self._analyzing_finished = True
html_after_timeouts = self.mainFrame().toHtml()
response_url = self.mainFrame().url().toString()
self.mainFrame().setHtml(None)
self._new_clickables.extend(clickables)
self._new_clickables.extend(elements_with_event_properties)
self._new_clickables = purge_dublicates(self._new_clickables)
response_code = None
try:
response_code = self.response_code[url_to_request]
except KeyError:
response_code = 200
if response_code is None:
response_code = 200
try:
current_page = WebPage(self.parent().get_next_page_id(), response_url, html_after_timeouts)
except AttributeError: # Attacker don't need this function...
current_page = WebPage(42, response_url, html_after_timeouts)
current_page.timing_requests = self._timing_requests
current_page.clickables = self._new_clickables
current_page.links = links
current_page.forms = forms
if base_url is not None and base_url != "":
current_page.base_url = base_url
return response_code, current_page
def analyze(self,
url_to_request,
timeout=10,
current_depth=None,
method="GET",
data={}):
try:
url_to_request = url_to_request.toString()
except AttributeError:
url_to_request = url_to_request
logging.debug("Start analyzing the url {}...".format(url_to_request))
self._timing_requests = []
self._new_clickables = []
self._timeming_events = []
self._current_timeming_event = None
self._loading_complete = False
self._analyzing_finished = False
self.response_code = {}
if method == "GET":
self.mainFrame().load(QUrl(url_to_request))
else:
request = self.make_request(url_to_request)
data = self.post_data_to_array(data)
request.setRawHeader(
'Content-Type',
QByteArray('application/x-www-form-urlencoded'))
self.mainFrame().load(request, QNetworkAccessManager.PostOperation,
data)
t = 0
while (not self._loading_complete
and t < timeout): # Waiting for finish processing
self._wait(self.wait_for_processing)
t += self.wait_for_processing
videos = self.mainFrame().findAllElements("video")
if len(videos) > 0:
logging.debug("{} videos found... removing them")
for video in videos:
video.removeFromDocument()
overall_waiting_time = t
buffer = 250
while len(
self._timeming_events) > 0 and overall_waiting_time < timeout:
self._current_timeming_event = self._timeming_events.pop(
0) # Take the first event(ordered by needed time
self._waiting_for = self._current_timeming_event[
'event_type'] # Setting kind of event
waiting_time_in_milliseconds = (
self._current_timeming_event["time"] - overall_waiting_time
) # Taking waiting time and convert it from milliseconds to seconds
waiting_time_in_milliseconds = (
(waiting_time_in_milliseconds + buffer) / 1000.0)
if waiting_time_in_milliseconds < 0.0:
waiting_time_in_milliseconds = 0
self._wait(waiting_time_in_milliseconds
) # Waiting for 100 millisecond before expected event
overall_waiting_time += waiting_time_in_milliseconds
if overall_waiting_time < 0.5:
self._wait((0.5 - overall_waiting_time))
# Just for debugging
#f = open("text.txt", "w", encoding="utf-8")
#f.write(self.mainFrame().toHtml())
#f.close()
base_url = self.mainFrame().findFirstElement("base")
if base_url is not None:
base_url = base_url.attribute("href")
links, clickables = extract_links(self.mainFrame(), url_to_request)
forms = extract_forms(self.mainFrame())
elements_with_event_properties = property_helper(self.mainFrame())
self.mainFrame().evaluateJavaScript(self._property_obs_js)
self._wait(0.1)
self._analyzing_finished = True
html_after_timeouts = self.mainFrame().toHtml()
response_url = self.mainFrame().url().toString()
self.mainFrame().setHtml(None)
self._new_clickables.extend(clickables)
self._new_clickables.extend(elements_with_event_properties)
self._new_clickables = purge_dublicates(self._new_clickables)
response_code = None
try:
response_code = self.response_code[url_to_request]
except KeyError:
response_code = 200
if response_code is None:
response_code = 200
try:
current_page = WebPage(self.parent().get_next_page_id(),
response_url, html_after_timeouts)
except AttributeError: #Attacker don't need this function...
current_page = WebPage(42, response_url, html_after_timeouts)
current_page.timing_requests = self._timing_requests
current_page.clickables = self._new_clickables
current_page.links = links
current_page.forms = forms
if base_url is not None and base_url != "":
current_page.base_url = base_url
return response_code, current_page
def execute(self, webpage, timeout=5, element_to_click=None, xhr_options=XHRBehavior.ObserveXHR, pre_clicks=[]):
logging.debug(
"EventExecutor test started on {}...".format(webpage.url) + " with " + element_to_click.toString())
self._analyzing_finished = False
self._loading_complete = False
self.xhr_options = xhr_options
self.element_to_click = None
self.ajax_requests = []
self._new_url = None
self.timeming_events = None
self._capturing_ajax = False
self._new_clickables = []
self.element_to_click = element_to_click
self.popup = None
self.mainFrame().setHtml(webpage.html, QUrl(webpage.url))
target_tag = element_to_click.dom_address.split("/")
target_tag = target_tag[-1]
if target_tag in ['video']:
return EventResult.UnsupportedTag, None
t = 0.0
while (not self._loading_complete and t < timeout ): # Waiting for finish processing
self._wait(0.1)
t += 0.1
if not self._loading_complete:
logging.debug("Timeout occurs while initial page loading...")
return EventResult.ErrorWhileInitialLoading, None
# Prepare Page for clicking...
self._wait(0.1)
for click in pre_clicks:
pre_click_elem = None
logging.debug("Click on: " + click.toString())
if click.id != None and click.id != "":
pre_click_elem = self.search_element_with_id(click.id)
if click.html_class != None and pre_click_elem == None:
pre_click_elem = self.search_element_with_class(click.html_class, click.dom_address)
if pre_click_elem == None:
pre_click_elem = self.search_element_without_id_and_class(click.dom_address)
if pre_click_elem is None:
logging.debug("Preclicking element not found")
return EventResult.PreviousClickNotFound, None
if "javascript:" not in click.event:
js_code = click.event
if js_code[0:2] == "on":
js_code = js_code[2:] # if event beginns with on, escape it
js_code = "Simulate." + js_code + "(this);"
pre_click_elem.evaluateJavaScript(js_code) # Waiting for finish processing
else:
pre_click_elem.evaluateJavaScript(click.event[len("javascript:"):])
self._wait(self.wait_for_event)
is_key_event = False
# Now execute the target event
if "javascript:" not in element_to_click.event:
self._url_changed = False
js_code = element_to_click.event
if js_code[0:2] == "on":
js_code = js_code[2:] # if event begins with on, escape it
if js_code in self.key_events:
is_key_event = True
random_char = random.choice(string.ascii_letters)
js_code = "Simulate." + js_code + "(this, '" + random_char + "');"
else:
js_code = "Simulate." + js_code + "(this);"
else:
js_code = element_to_click.event[len("javascript:"):]
self.mainFrame().evaluateJavaScript(
self._addEventListener) # This time it is here, because I dont want to have the initial addings
real_clickable = None
if element_to_click.id != None and element_to_click.id != "":
real_clickable = self.search_element_with_id(element_to_click.id)
if element_to_click.html_class != None and real_clickable == None:
real_clickable = self.search_element_with_class(element_to_click.html_class, element_to_click.dom_address)
if real_clickable == None:
real_clickable = self.search_element_without_id_and_class(element_to_click.dom_address)
if real_clickable is None:
logging.debug("Target Clickable not found")
return EventResult.TargetElementNotFound, None
self._capturing_ajax = True
real_clickable.evaluateJavaScript(js_code)
self._wait(0.5)
self._capturing_ajax = False
links, clickables = extract_links(self.mainFrame(), webpage.url)
forms = extract_forms(self.mainFrame())
elements_with_event_properties = property_helper(self.mainFrame())
self.mainFrame().evaluateJavaScript(self._property_obs_js)
self._wait(0.1)
html = self.mainFrame().toHtml()
url = self.mainFrame().url().toString()
if is_key_event:
generator = KeyClickable(element_to_click, random_char)
else:
generator = element_to_click
if self._url_changed and self._new_url.toString() != webpage.url:
delta_page = DeltaPage(-1, self._new_url.toString(), html=None, generator=generator, parent_id=webpage.id,
cookiesjar=webpage.cookiejar)
self._analyzing_finished = True
self.mainFrame().setHtml(None)
return EventResult.URLChanged, delta_page
elif self.popup is not None:
logging.debug("Event creates Popup with Url: {}".format(self.popup.mainFrame().url().toString()))
popup_url = self.popup.mainFrame().url().toString()
delta_page = DeltaPage(-1, popup_url, html=None, generator=generator, parent_id=webpage.id)
self.popup = None
self._analyzing_finished = True
self.mainFrame().setHtml(None)
return EventResult.CreatesPopup, delta_page
else:
delta_page = DeltaPage(-1, webpage.url, html, generator=generator, parent_id=webpage.id,
cookiesjar=webpage.cookiejar)
delta_page.clickables = self._new_clickables # Set by add eventlistener code
delta_page.clickables.extend(clickables)
delta_page.clickables.extend(elements_with_event_properties)
delta_page.clickables = purge_dublicates(delta_page.clickables)
try:
delta_page.clickables.remove(self.element_to_click) # remove the clickable self...
except ValueError:
pass
delta_page.links = links
delta_page.forms = forms
delta_page.ajax_requests = self.ajax_requests
self._analyzing_finished = True
self.mainFrame().setHtml(None)
return EventResult.Ok, delta_page
def execute(self,
webpage,
timeout=5,
element_to_click=None,
xhr_options=XHRBehavior.ObserveXHR,
pre_clicks=[]):
logging.debug(
"EventExecutor test started on {}... with {} and preclicks {}".
format(
webpage.url, "None"
if element_to_click == None else element_to_click.toString(),
[x.toString() for x in pre_clicks]))
self._analyzing_finished = False
self._loading_complete = False
self.xhr_options = xhr_options
self.element_to_click = None
self.ajax_requests = []
self._new_url = None
self.timeming_events = None
self._capturing_ajax = False
self._new_clickables = []
self.element_to_click = element_to_click
self.popup = None
self.mainFrame().setHtml(webpage.html, QUrl(webpage.url))
target_tag = element_to_click.dom_address.split(
"/") if element_to_click != None else "xyz"
target_tag = target_tag[-1]
if target_tag in ['video']:
return EventResult.UnsupportedTag, None
cbData = {
"element_to_click": element_to_click,
"pre_clicks": pre_clicks,
"webpage": webpage,
"xhr_options": xhr_options,
"timeout": timeout,
"self": self,
}
t = 0.0
while (not self._loading_complete
and t < timeout): # Waiting for finish processing
logging.debug("Waiting to load")
self._wait(0.1)
t += 0.1
if not self._loading_complete:
logging.debug("Timeout occurs while initial page loading...")
return EventResult.ErrorWhileInitialLoading, None
# Prepare Page for clicking...
self._wait(0.1)
for click in pre_clicks:
pre_click_elem = None
logging.debug("Click on: " + click.toString())
if click.id != None and click.id != "":
pre_click_elem = self.search_element_with_id(click.id)
if click.html_class != None and pre_click_elem == None:
pre_click_elem = self.search_element_with_class(
click.html_class, click.dom_address)
if pre_click_elem == None:
pre_click_elem = self.search_element_without_id_and_class(
click.dom_address)
if pre_click_elem is None:
logging.debug("Preclicking element not found")
return EventResult.PreviousClickNotFound, None
if "javascript:" not in click.event:
js_code = click.event
if js_code[0:2] == "on":
js_code = js_code[
2:] # if event beginns with on, escape it
js_code = "Simulate." + js_code + "(this);"
pre_click_elem.evaluateJavaScript(
js_code) # Waiting for finish processing
else:
pre_click_elem.evaluateJavaScript(
click.event[len("javascript:"):])
self._wait(self.wait_for_event)
is_key_event = False
real_clickable = None
# Now execute the target event
if element_to_click != None:
if "javascript:" not in element_to_click.event:
self._url_changed = False
js_code = element_to_click.event
if js_code[0:2] == "on":
js_code = js_code[2:] # if event begins with on, escape it
if js_code in self.key_events:
is_key_event = True
random_char = random.choice(string.ascii_letters)
js_code = "Simulate." + js_code + "(this, '" + random_char + "');"
else:
js_code = "Simulate." + js_code + "(this);"
else:
js_code = element_to_click.event[len("javascript:"):]
self.mainFrame().evaluateJavaScript(
self._addEventListener
) # This time it is here, because I dont want to have the initial addings
real_clickable = None
if element_to_click.id != None and element_to_click.id != "":
real_clickable = self.search_element_with_id(
element_to_click.id)
if element_to_click.html_class != None and real_clickable == None:
real_clickable = self.search_element_with_class(
element_to_click.html_class, element_to_click.dom_address)
if real_clickable == None:
real_clickable = self.search_element_without_id_and_class(
element_to_click.dom_address)
if real_clickable is None:
logging.debug("Target Clickable not found")
self.triggerAfterClicksHandler(
"afterclicks", {
"data": cbData,
"errorcode": EventResult.TargetElementNotFound
})
return EventResult.TargetElementNotFound, None
self._capturing_ajax = True
real_clickable.evaluateJavaScript(js_code)
self._wait(0.5)
self._capturing_ajax = False
links, clickables = extract_links(self.mainFrame(), webpage.url)
forms = extract_forms(self.mainFrame())
elements_with_event_properties = property_helper(self.mainFrame())
self.mainFrame().evaluateJavaScript(self._property_obs_js)
self._wait(0.1)
html = self.mainFrame().toHtml()
url = self.mainFrame().url().toString()
if is_key_event:
generator = KeyClickable(element_to_click, random_char)
else:
generator = element_to_click
if self._url_changed and self._new_url.toString() != webpage.url:
delta_page = DeltaPage(-1,
self._new_url.toString(),
html=None,
generator=generator,
parent_id=webpage.id,
cookiesjar=webpage.cookiejar)
self._analyzing_finished = True
self.triggerAfterClicksHandler("afterclicks", {
"data": cbData,
"errorcode": EventResult.URLChanged
})
self.mainFrame().setHtml(None)
return EventResult.URLChanged, delta_page
elif self.popup is not None:
logging.debug("Event creates Popup with Url: {}".format(
self.popup.mainFrame().url().toString()))
popup_url = self.popup.mainFrame().url().toString()
delta_page = DeltaPage(-1,
popup_url,
html=None,
generator=generator,
parent_id=webpage.id)
self.popup = None
self._analyzing_finished = True
self.triggerAfterClicksHandler(
"afterclicks", {
"data": cbData,
"errorcode": EventResult.CreatesPopup
})
self.mainFrame().setHtml(None)
return EventResult.CreatesPopup, delta_page
else:
delta_page = DeltaPage(-1,
webpage.url,
html,
generator=generator,
parent_id=webpage.id,
cookiesjar=webpage.cookiejar)
delta_page.clickables = self._new_clickables # Set by add eventlistener code
delta_page.clickables.extend(clickables)
delta_page.clickables.extend(elements_with_event_properties)
delta_page.clickables = purge_dublicates(delta_page.clickables)
try:
delta_page.clickables.remove(
self.element_to_click) # remove the clickable self...
except ValueError:
pass
delta_page.links = links
delta_page.forms = forms
delta_page.ajax_requests = self.ajax_requests
self._analyzing_finished = True
self.triggerAfterClicksHandler("afterclicks", {
"data": cbData,
"errorcode": EventResult.Ok
})
self.mainFrame().setHtml(None)
return EventResult.Ok, delta_page
def submit_form(self, form, webpage, data=dict(), timeout=5):
logging.debug("FormHandler on Page: {} started...".format(webpage.url))
self._loading_complete = False
self._analyzing_finished = False
try:
url = webpage.url.toString()
except AttributeError:
url = webpage.url
self.mainFrame().setHtml(webpage.html, QUrl(url))
self._new_clickables = []
t = 0.0
while not self._loading_complete and t < timeout: # Waiting for finish processing
self._wait(0.1)
t += 0.1
if not self._loading_complete:
logging.debug("Timeout occurs while initial page loading...")
return EventResult.ErrorWhileInitialLoading, None
target_form = None
p_forms = self.mainFrame().findAllElements("form")
for tmp_form in p_forms:
path = tmp_form.evaluateJavaScript("getXPath(this)")
if path == form.dom_address:
target_form = tmp_form
break
if target_form is None:
return EventResult.TargetElementNotFound, None
for elem in form.parameter: #Iterate through abstract form representation
if elem.name in data: #Check if we have the data we must set
elem_found = False # Indicates if we found the element in the html
value_to_set = data[elem.name]
for tmp in target_form.findAll(
elem.tag
): #Locking in the target form, if we found the element we have to set
if tmp.attribute(
"name"
) == elem.name: # Has the current element in the html the same name as our data
tmp.evaluateJavaScript("this.value = '" +
value_to_set + "';")
elem_found = True
break
if not elem_found:
return EventResult.TargetElementNotFound, None
# Now we should have set all known parameters, next click the submit button
q_submit_button = None
if "submit" in form.toString():
inputs = target_form.findAll("input") + target_form.findAll(
"button")
for el in inputs:
if el.attribute("type") == "submit":
q_submit_button = el
break
#q_submit_button.evaluateJavaScript("this.id='oxyfrymbel'")
else:
logging.debug(form.toString())
if q_submit_button is None:
inputs = target_form.findAll("button")
q_submit_button = None
if len(inputs) > 1:
logging.debug("Cannot locate login button...")
return EventResult.TargetElementNotFound, None
elif len(inputs) == 1:
q_submit_button = inputs[0]
method = target_form.attribute("onsubmit")
if method is not None and method != "":
js_code_snippets = method.split(";")
for snippet in js_code_snippets:
if "return" in snippet or snippet == "":
logging.debug("Ignoring snippet: {}".format(snippet))
continue
logging.debug("Eval: {}".format(snippet + ";"))
self.mainFrame().evaluateJavaScript(snippet + ";")
self._wait(3)
self.mainFrame().evaluateJavaScript(self._addEventListener)
self._wait(3)
else:
#TODO: Implement way for sending forms without onsubmit-method
# check between: target_form.evaluateJavaScript("Simulate or document.?form?.submit())
# or submit_button click
if q_submit_button is not None:
logging.debug("Click on submit button...")
q_submit_button.evaluateJavaScript("Simulate.click(this);")
self._wait(3)
else:
logging.debug("Trigger submit event on form...")
target_form.evaluateJavaScript("Simulate.submit(this);")
self._wait(3)
links, clickables = extract_links(self.mainFrame(), url)
forms = extract_forms(self.mainFrame())
html = self.mainFrame().toHtml()
#f = open("html.txt", "w", encoding="utf-8")
#f.write(html)
#f.close()
self.mainFrame().setHtml(None)
self._new_clickables.extend(clickables)
self._new_clickables = purge_dublicates(self._new_clickables)
return EventResult.Ok, html, self._new_clickables, forms, links, []
