class MyPanel(wx.Panel):
"""The GUI of the tool."""
def __init__(self, parent):
wx.Panel.__init__(self, parent)
self.frame = parent
# start Chrome webdriver
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
# chrome_options.add_argument('--auto-open-devtools-for-tabs')
caps = DesiredCapabilities.CHROME
caps['goog:loggingPrefs'] = {'performance': 'INFO'}
chrome_options.add_experimental_option('perfLoggingPrefs',
{'enablePage': True})
self.driver = webdriver.Chrome(options=chrome_options,
desired_capabilities=caps)
self.driver.execute_cdp_cmd('Network.enable', {})
self.driver.execute_cdp_cmd('Network.setCacheDisabled',
{'cacheDisabled': True})
self.driver.set_window_size(650, 750)
self.driver.set_window_position(0, 0)
self.main_sizer = wx.BoxSizer(wx.VERTICAL)
original_options = webdriver.ChromeOptions()
original_options.add_argument('--proxy-server=' + REMOTE_PROXY_IP +
':8082')
self.original = webdriver.Chrome(options=original_options)
self.original.set_window_size(650, 750)
self.original.set_window_position(650, 0)
# TextCtrl for user to input URL of site to analyze
self.url_input = wx.TextCtrl(self, style=wx.TE_LEFT)
self.url_input.SetValue("http://yasirzaki.net/")
self.url_input.Bind(wx.EVT_KEY_DOWN, self.on_key_press)
self.main_sizer.Add(self.url_input,
flag=wx.EXPAND | wx.TOP | wx.LEFT | wx.RIGHT,
border=25)
# StaticText field for error messages
self.err_msg = wx.StaticText(self, label="")
self.main_sizer.Add(self.err_msg, flag=wx.LEFT, border=25)
analyze_btn = wx.Button(self, label='Analyze page')
analyze_btn.Bind(wx.EVT_BUTTON, self.on_button_press)
self.main_sizer.Add(analyze_btn, flag=wx.ALL | wx.CENTER, border=5)
hbox = wx.BoxSizer(wx.HORIZONTAL)
self.scripts_panel = ScrolledPanel(self, size=(375, 550))
self.scripts_panel.SetupScrolling()
hbox.Add(self.scripts_panel)
self.content_panel = ScrolledPanel(self, size=(375, 550))
self.content_panel.SetupScrolling()
hbox.Add(self.content_panel, flag=wx.CENTER, border=5)
self.main_sizer.Add(hbox, flag=wx.CENTER | wx.BOTTOM, border=25)
hbox = wx.BoxSizer(wx.HORIZONTAL)
self.apply_btn = wx.Button(self, label='Apply Selection')
self.apply_btn.Bind(wx.EVT_BUTTON, self.on_button_press)
self.apply_btn.SetToolTip('Preview changes in the browser window.')
self.apply_btn.Hide()
hbox.Add(self.apply_btn, border=5)
self.save_btn = wx.Button(self, label='Save and load simplified page')
self.save_btn.Bind(wx.EVT_BUTTON, self.on_button_press)
self.save_btn.SetToolTip(
'Save changes in new folder and push to the remote proxy.')
self.save_btn.Hide()
hbox.Add(self.save_btn, border=5)
self.diff_btn = wx.Button(self, label='Get diff')
self.diff_btn.Bind(wx.EVT_BUTTON, self.on_button_press)
self.diff_btn.SetToolTip(
'Print diff before and after changes to terminal window.')
self.diff_btn.Hide()
hbox.Add(self.diff_btn, border=5)
self.main_sizer.Add(hbox, flag=wx.BOTTOM | wx.CENTER, border=25)
self.SetSizer(self.main_sizer)
self.url = self.url_input.GetValue()
self.suffix = "?JSTool=none"
self.script_sizer = wx.BoxSizer(wx.VERTICAL)
self.script_buttons = []
self.choice_boxes = []
self.number_of_buttons = 0
self.blocked_urls = []
self.content_panel.Hide()
self.content_text = ExpandoTextCtrl(self.content_panel,
size=(375, 275),
style=wx.TE_READONLY)
self.content_text.SetValue("Script code")
self.Bind(EVT_ETC_LAYOUT_NEEDED, None, self.content_text)
self.content_sizer = wx.BoxSizer(wx.VERTICAL)
self.content_sizer.Add(self.content_text, flag=wx.CENTER)
self.content_panel.SetSizer(self.content_sizer)
self.script_tree = AnyNode(id='root')
self.images = {}
self.yasir = {}
def on_button_press(self, event):
"""Handle wx.Button press."""
btn = event.GetEventObject()
if btn.GetLabel() == 'Analyze page':
self.analyze()
elif btn == self.diff_btn:
self.on_diff_press()
elif btn == self.apply_btn:
self.on_apply_press()
elif btn == self.save_btn:
self.on_save()
def on_key_press(self, event):
"""Handle keyboard input."""
keycode = event.GetKeyCode()
if keycode == wx.WXK_RETURN or keycode == wx.WXK_NUMPAD_ENTER:
self.analyze()
else:
event.Skip()
def add_button(self, script, index, depth, vector): # copies
"""Add script to self.script_buttons at index and update display."""
hbox = wx.BoxSizer(wx.HORIZONTAL)
hbox.AddSpacer(depth * 25)
# Create button
# if copies > 1: do something to differentiate it
self.script_buttons.insert(
index,
wx.CheckBox(self.scripts_panel, label=script.split("/")[-1][:9]))
self.script_buttons[index].myname = script
self.script_buttons[index].Bind(wx.EVT_CHECKBOX, self.on_script_press)
self.script_buttons[index].SetToolTip(script)
hbox.Add(self.script_buttons[index], flag=wx.ALL, border=5)
self.number_of_buttons += 1
# Create combobox
# choice_box = wx.ComboBox(self.scripts_panel, value="", style=wx.CB_READONLY, choices=(
# "", "critical", "non-critical", "replaceable"))
# choice_box.Bind(wx.EVT_COMBOBOX, self.on_choice)
# choice_box.index = len(self.choice_boxes)
# self.choice_boxes.insert(index, choice_box)
# hbox.Add(choice_box, flag=wx.ALL, border=5)
# self.number_of_buttons += 1
# Add labels
if script[:6] != 'script' and vector is not None:
category = ML_MODEL.predict([pandas.Series(vector)]).item(0)
confidence = np.amax(
ML_MODEL.predict_proba([pandas.Series(vector)]))
self.script_buttons[index].category = category
self.script_buttons[index].confidence = confidence
text = str(category) + ": " + str(int(confidence * 100)) + "%"
label = wx.StaticText(self.scripts_panel,
label=text,
style=wx.BORDER_RAISED)
label.SetBackgroundColour(tuple(CATEGORIES[category]['color']))
tool_tip = CATEGORIES[category]['description']
label.SetToolTip(tool_tip)
self.script_buttons[index].label = label
hbox.Add(label, flag=wx.ALL, border=5)
self.yasir[script] = self.script_buttons[index]
self.script_sizer.Insert(index, hbox)
self.frame.frame_sizer.Layout()
def format_src(self, src: str):
"""Return formatted src string to be requested."""
if src[:4] != "http":
if src[0] == "/":
if src[1] == "/":
src = "https:" + src
else:
src = self.url + src[1:]
else:
src = self.url + src
return src
def block_all_scripts(self):
"""Adds all scripts in self.script_tree to self.blocked_urls."""
self.blocked_urls.clear()
for node in PreOrderIter(self.script_tree):
if node.id[:6] != "script" and not node.is_root:
self.blocked_urls.append(node.id)
def wait_for_load(self):
"""Wait for page source to stop changing."""
html = self.driver.page_source
time.sleep(WAIT_LOAD_TIME)
while html != self.driver.page_source:
html = self.driver.page_source
time.sleep(WAIT_LOAD_TIME)
def analyze(self):
"""Do everything."""
def reset_display():
# Reset display
self.suffix = "?JSTool=none"
self.script_buttons.clear()
self.choice_boxes.clear()
self.number_of_buttons = 0
# self.diff_btn.Show()
self.apply_btn.Show()
self.save_btn.Show()
self.content_panel.Show()
self.content_text.SetValue("Script code")
while self.script_sizer.GetChildren():
self.script_sizer.Hide(0)
self.script_sizer.Remove(0)
self.images.clear()
def get_index_html():
# Get index.html from remote proxy
return get_resource(self.url)
def parse_html(html: str):
# Add index.html scripts to self.script_tree
cnt = 1
if not html:
return
while "<script" in html:
src = ""
script_name = "script" + str(cnt)
start_index = html.find("<script")
end_index = html.find("</script>")
text = html[start_index:end_index + 9]
new_node = AnyNode(id=script_name,
parent=self.script_tree,
content=text,
vector=extract_features(text),
count=1)
if ' src="' in text: # BeautifulSoup turns all single quotes into double quotes
src = text.split(' src="')[1].split('"')[0]
src = self.format_src(src)
try:
node = anytree.cachedsearch.find(
self.script_tree, lambda node: node.id == src)
except anytree.search.CountError:
logging.warning(
'multiple possible parents: more than one node with id = %s',
src)
if node:
node.parent = new_node
html = html.replace(text, "\n<!--" + script_name + "-->\n")
cnt += 1
def create_buttons():
# Add checkboxes to display
# Check all
self.add_button('Check all', 0, 1, None)
index = 1
# All other script checkboxes
for node in PreOrderIter(self.script_tree):
if node.is_root:
continue
node.button = index
# vector = extract_features(node.content)
self.add_button(node.id, index, node.depth,
get_attribute(node, 'vector')) # node.count
checkbox = self.script_buttons[index]
if (get_attribute(checkbox, 'confidence') is not None
and get_attribute(
checkbox, 'confidence') < CONFIDENCE_THRESHOLD):
# run clustering if confidence less than threshold
checkbox.category = CLUSTER.predict(script=str(
node.content),
preprocess=True)
label = get_attribute(checkbox, 'label')
if label:
label.SetLabel(checkbox.category)
label.SetBackgroundColour(
tuple(CATEGORIES[checkbox.category]['color']))
label.SetToolTip(
CATEGORIES[checkbox.category]['description'])
if get_attribute(checkbox,
'category') not in BLOCKED_CATEGORIES:
# ads / marketing scripts disabled by default
try:
if node.id[:6] != "script":
self.blocked_urls.remove(node.id)
except ValueError:
logging.debug("Could not remove %s from blocked urls",
node.id)
self.check_boxes(True, node)
index += 1
self.scripts_panel.SetSizer(self.script_sizer)
self.frame.frame_sizer.Layout()
def functional_dependency():
# functional dependencies?
try:
tmp_dep = perf.get_dependency(self.url)
# tmp_dep = [['https://ws.sharethis.com/button/async-buttons.js', 'https://www.google-analytics.com/analytics.js', 'https://ws.sharethis.com/button/buttons.js'], ['https://www.googletagmanager.com/gtm.js?id=GTM-WBDQQ5', 'https://www.googleadservices.com/pagead/conversion_async.js'], ['https://www.unicef.org/sites/default/files/js/js_B7pS3ddmNLFYOJi3j28odiodelMu-EhaOeKlHZ8E6y0.js', 'https://www.unicef.org/themes/custom/unicef/assets/src/js/init-blazy.js?v=1.x', 'https://www.unicef.org/sites/default/files/js/js_dWWS6YNlsZWmXLboSy3PIiSD_Yg3sRxwjbMb52mdNyw.js', 'https://www.unicef.org/sites/default/files/js/js_cLlwgRdoiVfjtFxLqlXX-aVbv3xxfX_uMCsn7iJqNpA.js']]
print("\n\n-------- DEPENDENCY LABELS CHANGED --------")
mapping = {'non-critical': 0, 'translatable': 1, 'critical': 2}
mapping2 = {
0: 'non-critical',
1: 'translatable',
2: 'critical'
}
for a in tmp_dep:
tmp_label = 0
for i in a:
if i not in self.yasir or self.yasir[
i].category not in mapping:
continue
if mapping[self.yasir[i].category] > tmp_label:
tmp_label = mapping[self.yasir[i].category]
for i in a:
if i not in self.yasir or self.yasir[
i].category not in mapping:
continue
if self.yasir[i].category != mapping2[tmp_label]:
print("****", i, mapping2[tmp_label],
self.yasir[i].category)
print("\n\n")
except RuntimeError:
pass
def display_loading_message():
# Never managed to get this part to display before spinning wheel of death
self.err_msg.SetForegroundColour((0, 0, 0))
self.err_msg.SetLabel("Loading page... please wait")
self.Update()
def similarity():
# Print script pairs in self.script_tree with Jaccard similarity > SIMILARITY_THRESHOLD
names = []
scripts = []
for node in PreOrderIter(self.script_tree):
if node.is_root:
continue
names.append(node.id)
scripts.append(str(node.content))
results = similarity_comparison(scripts, SIMILARITY_THRESHOLD)
if results:
print("---" * 20)
print('scripts with similarity > %.2f' % SIMILARITY_THRESHOLD)
for tup in results:
print('%s %s %.2f' % (names[tup[0]], names[tup[1]], tup[2]))
def compare_image_sizes(images):
# Print difference in original and rendered image sizes for image URLs in images
for url in images:
if url[:4] == 'data':
# URI rather than URL
url = url.partition(';')[-1]
body = url.partition(',')[-1]
if url[:6] == 'base64':
body = base64.b64decode(body)
else:
body = get_resource(url)
try:
stream = BytesIO(body)
except TypeError:
logging.warning("body in %s, not in bytes", type(body))
stream = BytesIO(body.encode(ENCODING))
try:
width, height = get_image_size_from_bytesio(
stream, DEFAULT_BUFFER_SIZE)
self.images[url] = {}
self.images[url]['ow'] = width
self.images[url]['oh'] = height
except UnknownImageFormat as error:
logging.exception(str(error))
except struct.error as error:
logging.error(str(error))
for img in self.driver.find_elements_by_tag_name('img'):
url = img.get_attribute('src')
if url not in self.images.keys():
self.images[url] = {}
self.images[url]['rw'] = img.size['width']
self.images[url]['rh'] = img.size['height']
logging.info("---" * 20)
logging.info("potential image improvements:")
for url, dimensions in self.images.items():
if len(dimensions.keys()) == 4:
# Successfully parsed original and rendered dimensions
logging.info(url)
logging.info("original: %d x %d", dimensions['ow'],
dimensions['oh'])
logging.info("rendered: %d x %d", dimensions['rw'],
dimensions['rh'])
display_loading_message()
# Reset values
self.url = self.url_input.GetValue()
if self.url[-1] != "/":
self.url = self.url + "/"
if not self.url:
return
reset_display()
self.script_tree = AnyNode(id=self.url)
try:
file_path = PATH + "/reports/" + self.url.split("/")[2]
if not os.path.exists(file_path):
os.mkdir(file_path)
with open(file_path + "/script_tree.txt", 'r') as f:
logging.debug('importing script tree...')
importer = JsonImporter()
self.script_tree = importer.read(f)
with open(file_path + "/images.json", 'r') as f:
images = json.load(f)
except FileNotFoundError:
logging.debug('script tree does not yet exist, building now')
# Get original page and parse external scripts
self.driver.execute_cdp_cmd('Network.setBlockedURLs', {'urls': []})
epoch_in_milliseconds = time.time() * 1000
try:
self.driver.get(self.url)
self.err_msg.SetLabel("")
except InvalidArgumentException as exception:
self.err_msg.SetForegroundColour((255, 0, 0)) # make text red
self.err_msg.SetLabel(str(exception))
return
self.wait_for_load()
self.script_tree = AnyNode(id=self.url)
scripts, images = self.parse_log(epoch_in_milliseconds)
for script in scripts:
# pylint: disable=undefined-loop-variable
# pylint: disable=cell-var-from-loop
parent = anytree.cachedsearch.find(
self.script_tree,
lambda node: node.id == self.format_src(script['parent']))
# Check if this node already exists
node = anytree.cachedsearch.find(
self.script_tree,
lambda node: node.id == self.format_src(script['url']))
if node and node.parent == parent:
logging.warning('duplicate script! %s',
self.format_src(script['url']))
node.count += 1
else:
AnyNode(id=self.format_src(script['url']),
parent=parent,
content=script['content'],
vector=extract_features(script['content']),
count=1)
# Check image differences
compare_image_sizes(images)
# Parse inline scripts
html = get_index_html()
parse_html(html)
# self.print_scripts()
# Export script tree
logging.debug('exporting script tree...')
exporter = JsonExporter()
with open(
PATH + "/reports/" + self.url.split("/")[2] +
"/script_tree.json", "w") as f:
exporter.write(self.script_tree, f)
logging.debug('done')
# Export images
with open(
PATH + "/reports/" + self.url.split("/")[2] +
"/images.json", "w") as f:
json.dump(images, f)
# Check similarity
# similarity()
# Create buttons
self.block_all_scripts()
create_buttons()
# Print functional dependencies
# functional_dependency()
# Get page with all scripts removed
self.on_apply_press()
try:
self.original.get(self.url)
except InvalidArgumentException as e:
logging.error(e.what())
# Used for diff
# final_html = BeautifulSoup(self.driver.execute_script(
# "return document.getElementsByTagName('html')[0].innerHTML"), 'html.parser')
# file_stream = open("before.html", "w")
# file_stream.write(final_html.prettify())
# file_stream.close()
def on_check_all(self, toggle):
"""Handle 'Select All' checkbox toggle."""
self.suffix = "?JSTool="
for btn in self.script_buttons:
btn.SetValue(toggle)
if toggle and btn.myname[:6] == "script":
self.suffix += "_" + btn.myname[6:]
if toggle:
# Toggle all script buttons
self.blocked_urls.clear()
else:
# Untoggle all script buttons
self.block_all_scripts()
self.suffix += "none"
def on_apply_press(self):
"""Send request for page with changes."""
self.driver.execute_cdp_cmd('Network.setBlockedURLs',
{'urls': self.blocked_urls})
self.suffix = "?JSTool="
for btn in self.script_buttons:
if btn.GetValue() and btn.myname[:6] == "script":
self.suffix += "_" + btn.myname[6:]
if self.suffix == "?JSTool=":
self.suffix += "none"
self.driver.get(self.url + self.suffix)
self.err_msg.SetLabel("")
def on_script_press(self, event):
"""Handle script button press."""
name = event.GetEventObject().myname
toggle = event.GetEventObject().GetValue()
if name == 'Check all':
self.on_check_all(toggle)
return
node = anytree.cachedsearch.find(self.script_tree,
lambda node: node.id == name)
if not get_attribute(node, 'content'):
node.content = get_resource(node.id)
self.content_text.SetValue(name + "\n\n" + str(node.content))
self.check_boxes(toggle, node)
def check_boxes(self, toggle: bool, node: AnyNode):
"""
Check (toggle = true) or uncheck (toggle = false) boxes
corresponding to node while keeping dependencies intact.
All ancestors of node are also checked if node is checked,
and all children of node are also unchecked if node is unchecked.
"""
if toggle:
while node.depth > 1:
self.script_buttons[node.button].SetValue(True)
try:
self.blocked_urls.remove(node.id)
except ValueError:
logging.debug("Could not remove %s from blocked urls",
node.id)
node = node.parent
self.script_buttons[node.button].SetValue(True)
if node.id[:6] != "script":
try:
self.blocked_urls.remove(node.id)
except ValueError:
logging.debug("Could not remove %s from blocked urls",
node.id)
else:
for descendant in node.descendants:
self.script_buttons[descendant.button].SetValue(False)
self.blocked_urls.append(descendant.id)
if node.id[:6] != "script":
self.blocked_urls.append(node.id)
def on_diff_press(self):
"""Print diff to terminal."""
after = BeautifulSoup(
self.driver.execute_script(
"return document.getElementsByTagName('html')[0].innerHTML"),
'html.parser')
try:
file_stream = open("after.html", "r")
before = file_stream.read()
file_stream.close()
file_stream = open("before.html", "w")
file_stream.write(before)
file_stream.close()
before = BeautifulSoup(before, 'html.parser')
except IOError:
pass
file_stream = open("after.html", "w")
file_stream.write(after.prettify())
file_stream.close()
os.system(r"git diff --no-index before.html after.html")
# os.system(r"diff before.html after.html | sed '/<!--script/,/<\/script>/d'")
def on_save(self):
"""Generate report and save in reports folder."""
if not os.path.exists(PATH + "/reports"):
os.mkdir(PATH + "/reports")
file_path = PATH + "/reports/" + self.url.split("/")[2]
if not os.path.exists(file_path):
os.mkdir(file_path)
logging.info("Writing script files...")
critical = open(file_path + '/critical.txt', 'w')
noncritical = open(file_path + '/noncritical.txt', 'w')
webalmanac = open(file_path + '/webalmanac.txt', 'w')
# labels = open(PATH + "/reports/labels.csv", 'a')
for node in PreOrderIter(self.script_tree):
if node.is_root or node.id[:6] == 'script':
continue
checkbox = self.script_buttons[get_attribute(node, 'button')]
if checkbox.GetValue():
critical.write(node.id + "\n")
else:
noncritical.write(node.id + "\n")
label = get_attribute(checkbox, 'label')
if label and label.GetLabel() != 'critical' and label.GetLabel(
) != 'non-critical':
webalmanac.write(node.id + "\n")
webalmanac.write(label.GetLabel() + "\n")
# if checkbox.GetValue():
# labels.write(str(node.vector.to_list()) + "," +
# labels.write(str(node.vector) + "," +
# label.GetLabel() + ",critical\n")
# else:
# labels.write(str(node.vector.to_list()) + "," +
# labels.write(str(node.vector) + "," +
# label.GetLabel() + ",non-critical\n")
critical.close()
noncritical.close()
webalmanac.close()
# labels.close()
logging.info("Writing index file...")
index = open(file_path + '/index.html', 'w')
# pickle.dump(get_resource(self.url + self.suffix) + "\n", index)
index.write(get_resource(self.url + self.suffix))
index.close()
logging.info("Writing images file...")
images = open(file_path + '/images.txt', 'w')
for url, dimensions in self.images.items():
images.write(url + "\n")
if 'ow' in dimensions and 'oh' in dimensions:
images.write("original: %d x %d\n" %
(dimensions['ow'], dimensions['oh']))
if 'rw' in dimensions and 'rh' in dimensions:
images.write("rendered: %d x %d\n" %
(dimensions['rw'], dimensions['rh']))
images.close()
self.err_msg.SetForegroundColour((0, 0, 0))
logging.info("Report generated in %s", file_path)
# Send report to proxy
multipart_form_data = {
'html_content': (open(file_path + '/index.html', 'rb')),
'blocked_URLs': (open(file_path + '/noncritical.txt', 'rb')),
'images': (open(file_path + '/images.txt', 'rb')),
'url': self.url,
}
response = requests.post("http://" + REMOTE_PROXY_IP +
":9000/JSCleaner/JSAnalyzer.py",
files=multipart_form_data)
if response.status_code == 200:
self.err_msg.SetLabel("Report sent to external proxy")
else:
self.err_msg.SetLabel(
"Report could not be sent - report generated in %s" %
file_path)
logging.error(response.status_code)
logging.error(response.headers)
logging.debug(response.text)
# Load simplified page from proxy
simplified_options = webdriver.ChromeOptions()
simplified_options.add_argument('--proxy-server=' + REMOTE_PROXY_IP +
':8083')
simplified = webdriver.Chrome(options=simplified_options)
simplified.set_window_size(600, 750)
simplified.set_window_position(650, 0)
simplified.get(self.url + '?JSCleaner')
input()
simplified.close()
def on_choice(self, event):
"""Handle choiceBox selection."""
choice_box = self.choice_boxes[event.GetEventObject().index]
colors = {
"": wx.Colour(255, 255, 255, 100),
"critical": wx.Colour(255, 0, 0, 100),
"non-critical": wx.Colour(0, 255, 0, 100),
"translatable": wx.Colour(0, 0, 255, 100)
}
choice_box.SetBackgroundColour(colors[choice_box.GetValue()])
def parse_log(self, epoch_in_milliseconds):
"""Return list of scripts requested since epoch_in_milliseconds."""
scripts = []
images = []
log = self.driver.get_log('performance')
log = log[bisect.bisect_left([entry['timestamp']
for entry in log], epoch_in_milliseconds
):]
log = [json.loads(entry['message'])['message'] for entry in log]
def is_script_request(message):
if message['method'] == 'Network.requestWillBeSent':
if message['params']['type'] == 'Script':
return True
return False
def is_image_request(message):
if message['method'] == 'Network.requestWillBeSent':
if message['params']['type'] == 'Image':
return True
return False
# def is_script_response(message):
# if message['method'] == 'Network.responseReceived':
# if 'javascript' in message['params']['response']['mimeType']:
# return True
# return False
def is_data_received(message):
if message['method'] == 'Network.dataReceived':
return True
return False
def get_request_info(message):
request_id = message['params']['requestId']
request_url = message['params']['request']['url']
initiator = message['params']['initiator']
if initiator['type'] == 'parser':
# from index.html as src, need to identify script number somehow...
# there are line numbers but are those usable?
initiator = initiator['url']
elif initiator['type'] == 'script':
# pick last thing in callFrames because first thing doesn't always have URL?
# need better understanding
# each script has its own ID... if only I could figure out how to use it
if initiator['stack']['callFrames']:
initiator = initiator['stack']['callFrames'][-1]['url']
return [request_id, request_url, initiator]
script_requests = []
# script_responses = []
image_requests = []
data_received = []
for message in log:
if is_script_request(message):
script_requests.append(message)
# elif is_script_response(message):
# script_responses.append(message['params']['requestId'])
elif is_image_request(message):
image_requests.append(message)
elif is_data_received(message):
data_received.append(message['params']['requestId'])
for request in script_requests:
request_id, url, initiator = get_request_info(request)
if request_id in data_received:
content = get_resource(url)
scripts.append({
'url': url,
'parent': initiator,
'content': content
})
for request in image_requests:
request_id, url, initiator = get_request_info(request)
if request_id in data_received:
images.append(url)
return (scripts, images)
def print_scripts(self):
"""Print script tree."""
print(RenderTree(self.script_tree).by_attr('id'))
print("---" * 20)
def print_blocked_scripts(self):
"""Print blocked URLs."""
print('BLOCKED SCRIPTS:')
for url in self.blocked_urls:
print("\t", url)
