class MyPanel(wx.Panel): """The GUI of the tool.""" def __init__(self, parent): wx.Panel.__init__(self, parent) self.frame = parent # start Chrome webdriver chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--proxy-server=%s' % PROXY) # chrome_options.add_argument('--auto-open-devtools-for-tabs') caps = DesiredCapabilities.CHROME caps['goog:loggingPrefs'] = {'performance': 'INFO'} chrome_options.add_experimental_option('perfLoggingPrefs', {'enablePage': True}) self.driver = webdriver.Chrome(options=chrome_options, desired_capabilities=caps) self.driver.execute_cdp_cmd('Network.enable', {}) self.driver.execute_cdp_cmd('Network.setCacheDisabled', {'cacheDisabled': True}) self.driver.set_window_size(650, 750) self.driver.set_window_position(0, 0) self.main_sizer = wx.BoxSizer(wx.VERTICAL) original_options = webdriver.ChromeOptions() original_options.add_argument('--proxy-server=' + REMOTE_PROXY_IP + ':8082') self.original = webdriver.Chrome(options=original_options) self.original.set_window_size(650, 750) self.original.set_window_position(650, 0) # TextCtrl for user to input URL of site to analyze self.url_input = wx.TextCtrl(self, style=wx.TE_LEFT) self.url_input.SetValue("http://yasirzaki.net/") self.url_input.Bind(wx.EVT_KEY_DOWN, self.on_key_press) self.main_sizer.Add(self.url_input, flag=wx.EXPAND | wx.TOP | wx.LEFT | wx.RIGHT, border=25) # StaticText field for error messages self.err_msg = wx.StaticText(self, label="") self.main_sizer.Add(self.err_msg, flag=wx.LEFT, border=25) analyze_btn = wx.Button(self, label='Analyze page') analyze_btn.Bind(wx.EVT_BUTTON, self.on_button_press) self.main_sizer.Add(analyze_btn, flag=wx.ALL | wx.CENTER, border=5) hbox = wx.BoxSizer(wx.HORIZONTAL) self.scripts_panel = ScrolledPanel(self, size=(375, 550)) self.scripts_panel.SetupScrolling() hbox.Add(self.scripts_panel) self.content_panel = ScrolledPanel(self, size=(375, 550)) self.content_panel.SetupScrolling() hbox.Add(self.content_panel, flag=wx.CENTER, border=5) self.main_sizer.Add(hbox, flag=wx.CENTER | wx.BOTTOM, border=25) hbox = wx.BoxSizer(wx.HORIZONTAL) self.apply_btn = wx.Button(self, label='Apply Selection') self.apply_btn.Bind(wx.EVT_BUTTON, self.on_button_press) self.apply_btn.SetToolTip('Preview changes in the browser window.') self.apply_btn.Hide() hbox.Add(self.apply_btn, border=5) self.save_btn = wx.Button(self, label='Save and load simplified page') self.save_btn.Bind(wx.EVT_BUTTON, self.on_button_press) self.save_btn.SetToolTip( 'Save changes in new folder and push to the remote proxy.') self.save_btn.Hide() hbox.Add(self.save_btn, border=5) self.diff_btn = wx.Button(self, label='Get diff') self.diff_btn.Bind(wx.EVT_BUTTON, self.on_button_press) self.diff_btn.SetToolTip( 'Print diff before and after changes to terminal window.') self.diff_btn.Hide() hbox.Add(self.diff_btn, border=5) self.main_sizer.Add(hbox, flag=wx.BOTTOM | wx.CENTER, border=25) self.SetSizer(self.main_sizer) self.url = self.url_input.GetValue() self.suffix = "?JSTool=none" self.script_sizer = wx.BoxSizer(wx.VERTICAL) self.script_buttons = [] self.choice_boxes = [] self.number_of_buttons = 0 self.blocked_urls = [] self.content_panel.Hide() self.content_text = ExpandoTextCtrl(self.content_panel, size=(375, 275), style=wx.TE_READONLY) self.content_text.SetValue("Script code") self.Bind(EVT_ETC_LAYOUT_NEEDED, None, self.content_text) self.content_sizer = wx.BoxSizer(wx.VERTICAL) self.content_sizer.Add(self.content_text, flag=wx.CENTER) self.content_panel.SetSizer(self.content_sizer) self.script_tree = AnyNode(id='root') self.images = {} self.yasir = {} def on_button_press(self, event): """Handle wx.Button press.""" btn = event.GetEventObject() if btn.GetLabel() == 'Analyze page': self.analyze() elif btn == self.diff_btn: self.on_diff_press() elif btn == self.apply_btn: self.on_apply_press() elif btn == self.save_btn: self.on_save() def on_key_press(self, event): """Handle keyboard input.""" keycode = event.GetKeyCode() if keycode == wx.WXK_RETURN or keycode == wx.WXK_NUMPAD_ENTER: self.analyze() else: event.Skip() def add_button(self, script, index, depth, vector): # copies """Add script to self.script_buttons at index and update display.""" hbox = wx.BoxSizer(wx.HORIZONTAL) hbox.AddSpacer(depth * 25) # Create button # if copies > 1: do something to differentiate it self.script_buttons.insert( index, wx.CheckBox(self.scripts_panel, label=script.split("/")[-1][:9])) self.script_buttons[index].myname = script self.script_buttons[index].Bind(wx.EVT_CHECKBOX, self.on_script_press) self.script_buttons[index].SetToolTip(script) hbox.Add(self.script_buttons[index], flag=wx.ALL, border=5) self.number_of_buttons += 1 # Create combobox # choice_box = wx.ComboBox(self.scripts_panel, value="", style=wx.CB_READONLY, choices=( # "", "critical", "non-critical", "replaceable")) # choice_box.Bind(wx.EVT_COMBOBOX, self.on_choice) # choice_box.index = len(self.choice_boxes) # self.choice_boxes.insert(index, choice_box) # hbox.Add(choice_box, flag=wx.ALL, border=5) # self.number_of_buttons += 1 # Add labels if script[:6] != 'script' and vector is not None: category = ML_MODEL.predict([pandas.Series(vector)]).item(0) confidence = np.amax( ML_MODEL.predict_proba([pandas.Series(vector)])) self.script_buttons[index].category = category self.script_buttons[index].confidence = confidence text = str(category) + ": " + str(int(confidence * 100)) + "%" label = wx.StaticText(self.scripts_panel, label=text, style=wx.BORDER_RAISED) label.SetBackgroundColour(tuple(CATEGORIES[category]['color'])) tool_tip = CATEGORIES[category]['description'] label.SetToolTip(tool_tip) self.script_buttons[index].label = label hbox.Add(label, flag=wx.ALL, border=5) self.yasir[script] = self.script_buttons[index] self.script_sizer.Insert(index, hbox) self.frame.frame_sizer.Layout() def format_src(self, src: str): """Return formatted src string to be requested.""" if src[:4] != "http": if src[0] == "/": if src[1] == "/": src = "https:" + src else: src = self.url + src[1:] else: src = self.url + src return src def block_all_scripts(self): """Adds all scripts in self.script_tree to self.blocked_urls.""" self.blocked_urls.clear() for node in PreOrderIter(self.script_tree): if node.id[:6] != "script" and not node.is_root: self.blocked_urls.append(node.id) def wait_for_load(self): """Wait for page source to stop changing.""" html = self.driver.page_source time.sleep(WAIT_LOAD_TIME) while html != self.driver.page_source: html = self.driver.page_source time.sleep(WAIT_LOAD_TIME) def analyze(self): """Do everything.""" def reset_display(): # Reset display self.suffix = "?JSTool=none" self.script_buttons.clear() self.choice_boxes.clear() self.number_of_buttons = 0 # self.diff_btn.Show() self.apply_btn.Show() self.save_btn.Show() self.content_panel.Show() self.content_text.SetValue("Script code") while self.script_sizer.GetChildren(): self.script_sizer.Hide(0) self.script_sizer.Remove(0) self.images.clear() def get_index_html(): # Get index.html from remote proxy return get_resource(self.url) def parse_html(html: str): # Add index.html scripts to self.script_tree cnt = 1 if not html: return while "<script" in html: src = "" script_name = "script" + str(cnt) start_index = html.find("<script") end_index = html.find("</script>") text = html[start_index:end_index + 9] new_node = AnyNode(id=script_name, parent=self.script_tree, content=text, vector=extract_features(text), count=1) if ' src="' in text: # BeautifulSoup turns all single quotes into double quotes src = text.split(' src="')[1].split('"')[0] src = self.format_src(src) try: node = anytree.cachedsearch.find( self.script_tree, lambda node: node.id == src) except anytree.search.CountError: logging.warning( 'multiple possible parents: more than one node with id = %s', src) if node: node.parent = new_node html = html.replace(text, "\n<!--" + script_name + "-->\n") cnt += 1 def create_buttons(): # Add checkboxes to display # Check all self.add_button('Check all', 0, 1, None) index = 1 # All other script checkboxes for node in PreOrderIter(self.script_tree): if node.is_root: continue node.button = index # vector = extract_features(node.content) self.add_button(node.id, index, node.depth, get_attribute(node, 'vector')) # node.count checkbox = self.script_buttons[index] if (get_attribute(checkbox, 'confidence') is not None and get_attribute( checkbox, 'confidence') < CONFIDENCE_THRESHOLD): # run clustering if confidence less than threshold checkbox.category = CLUSTER.predict(script=str( node.content), preprocess=True) label = get_attribute(checkbox, 'label') if label: label.SetLabel(checkbox.category) label.SetBackgroundColour( tuple(CATEGORIES[checkbox.category]['color'])) label.SetToolTip( CATEGORIES[checkbox.category]['description']) if get_attribute(checkbox, 'category') not in BLOCKED_CATEGORIES: # ads / marketing scripts disabled by default try: if node.id[:6] != "script": self.blocked_urls.remove(node.id) except ValueError: logging.debug("Could not remove %s from blocked urls", node.id) self.check_boxes(True, node) index += 1 self.scripts_panel.SetSizer(self.script_sizer) self.frame.frame_sizer.Layout() def functional_dependency(): # functional dependencies? try: tmp_dep = perf.get_dependency(self.url) # tmp_dep = [['https://ws.sharethis.com/button/async-buttons.js', 'https://www.google-analytics.com/analytics.js', 'https://ws.sharethis.com/button/buttons.js'], ['https://www.googletagmanager.com/gtm.js?id=GTM-WBDQQ5', 'https://www.googleadservices.com/pagead/conversion_async.js'], ['https://www.unicef.org/sites/default/files/js/js_B7pS3ddmNLFYOJi3j28odiodelMu-EhaOeKlHZ8E6y0.js', 'https://www.unicef.org/themes/custom/unicef/assets/src/js/init-blazy.js?v=1.x', 'https://www.unicef.org/sites/default/files/js/js_dWWS6YNlsZWmXLboSy3PIiSD_Yg3sRxwjbMb52mdNyw.js', 'https://www.unicef.org/sites/default/files/js/js_cLlwgRdoiVfjtFxLqlXX-aVbv3xxfX_uMCsn7iJqNpA.js']] print("\n\n-------- DEPENDENCY LABELS CHANGED --------") mapping = {'non-critical': 0, 'translatable': 1, 'critical': 2} mapping2 = { 0: 'non-critical', 1: 'translatable', 2: 'critical' } for a in tmp_dep: tmp_label = 0 for i in a: if i not in self.yasir or self.yasir[ i].category not in mapping: continue if mapping[self.yasir[i].category] > tmp_label: tmp_label = mapping[self.yasir[i].category] for i in a: if i not in self.yasir or self.yasir[ i].category not in mapping: continue if self.yasir[i].category != mapping2[tmp_label]: print("****", i, mapping2[tmp_label], self.yasir[i].category) print("\n\n") except RuntimeError: pass def display_loading_message(): # Never managed to get this part to display before spinning wheel of death self.err_msg.SetForegroundColour((0, 0, 0)) self.err_msg.SetLabel("Loading page... please wait") self.Update() def similarity(): # Print script pairs in self.script_tree with Jaccard similarity > SIMILARITY_THRESHOLD names = [] scripts = [] for node in PreOrderIter(self.script_tree): if node.is_root: continue names.append(node.id) scripts.append(str(node.content)) results = similarity_comparison(scripts, SIMILARITY_THRESHOLD) if results: print("---" * 20) print('scripts with similarity > %.2f' % SIMILARITY_THRESHOLD) for tup in results: print('%s %s %.2f' % (names[tup[0]], names[tup[1]], tup[2])) def compare_image_sizes(images): # Print difference in original and rendered image sizes for image URLs in images for url in images: if url[:4] == 'data': # URI rather than URL url = url.partition(';')[-1] body = url.partition(',')[-1] if url[:6] == 'base64': body = base64.b64decode(body) else: body = get_resource(url) try: stream = BytesIO(body) except TypeError: logging.warning("body in %s, not in bytes", type(body)) stream = BytesIO(body.encode(ENCODING)) try: width, height = get_image_size_from_bytesio( stream, DEFAULT_BUFFER_SIZE) self.images[url] = {} self.images[url]['ow'] = width self.images[url]['oh'] = height except UnknownImageFormat as error: logging.exception(str(error)) except struct.error as error: logging.error(str(error)) for img in self.driver.find_elements_by_tag_name('img'): url = img.get_attribute('src') if url not in self.images.keys(): self.images[url] = {} self.images[url]['rw'] = img.size['width'] self.images[url]['rh'] = img.size['height'] logging.info("---" * 20) logging.info("potential image improvements:") for url, dimensions in self.images.items(): if len(dimensions.keys()) == 4: # Successfully parsed original and rendered dimensions logging.info(url) logging.info("original: %d x %d", dimensions['ow'], dimensions['oh']) logging.info("rendered: %d x %d", dimensions['rw'], dimensions['rh']) display_loading_message() # Reset values self.url = self.url_input.GetValue() if self.url[-1] != "/": self.url = self.url + "/" if not self.url: return reset_display() self.script_tree = AnyNode(id=self.url) try: file_path = PATH + "/reports/" + self.url.split("/")[2] if not os.path.exists(file_path): os.mkdir(file_path) with open(file_path + "/script_tree.txt", 'r') as f: logging.debug('importing script tree...') importer = JsonImporter() self.script_tree = importer.read(f) with open(file_path + "/images.json", 'r') as f: images = json.load(f) except FileNotFoundError: logging.debug('script tree does not yet exist, building now') # Get original page and parse external scripts self.driver.execute_cdp_cmd('Network.setBlockedURLs', {'urls': []}) epoch_in_milliseconds = time.time() * 1000 try: self.driver.get(self.url) self.err_msg.SetLabel("") except InvalidArgumentException as exception: self.err_msg.SetForegroundColour((255, 0, 0)) # make text red self.err_msg.SetLabel(str(exception)) return self.wait_for_load() self.script_tree = AnyNode(id=self.url) scripts, images = self.parse_log(epoch_in_milliseconds) for script in scripts: # pylint: disable=undefined-loop-variable # pylint: disable=cell-var-from-loop parent = anytree.cachedsearch.find( self.script_tree, lambda node: node.id == self.format_src(script['parent'])) # Check if this node already exists node = anytree.cachedsearch.find( self.script_tree, lambda node: node.id == self.format_src(script['url'])) if node and node.parent == parent: logging.warning('duplicate script! %s', self.format_src(script['url'])) node.count += 1 else: AnyNode(id=self.format_src(script['url']), parent=parent, content=script['content'], vector=extract_features(script['content']), count=1) # Check image differences compare_image_sizes(images) # Parse inline scripts html = get_index_html() parse_html(html) # self.print_scripts() # Export script tree logging.debug('exporting script tree...') exporter = JsonExporter() with open( PATH + "/reports/" + self.url.split("/")[2] + "/script_tree.json", "w") as f: exporter.write(self.script_tree, f) logging.debug('done') # Export images with open( PATH + "/reports/" + self.url.split("/")[2] + "/images.json", "w") as f: json.dump(images, f) # Check similarity # similarity() # Create buttons self.block_all_scripts() create_buttons() # Print functional dependencies # functional_dependency() # Get page with all scripts removed self.on_apply_press() try: self.original.get(self.url) except InvalidArgumentException as e: logging.error(e.what()) # Used for diff # final_html = BeautifulSoup(self.driver.execute_script( # "return document.getElementsByTagName('html')[0].innerHTML"), 'html.parser') # file_stream = open("before.html", "w") # file_stream.write(final_html.prettify()) # file_stream.close() def on_check_all(self, toggle): """Handle 'Select All' checkbox toggle.""" self.suffix = "?JSTool=" for btn in self.script_buttons: btn.SetValue(toggle) if toggle and btn.myname[:6] == "script": self.suffix += "_" + btn.myname[6:] if toggle: # Toggle all script buttons self.blocked_urls.clear() else: # Untoggle all script buttons self.block_all_scripts() self.suffix += "none" def on_apply_press(self): """Send request for page with changes.""" self.driver.execute_cdp_cmd('Network.setBlockedURLs', {'urls': self.blocked_urls}) self.suffix = "?JSTool=" for btn in self.script_buttons: if btn.GetValue() and btn.myname[:6] == "script": self.suffix += "_" + btn.myname[6:] if self.suffix == "?JSTool=": self.suffix += "none" self.driver.get(self.url + self.suffix) self.err_msg.SetLabel("") def on_script_press(self, event): """Handle script button press.""" name = event.GetEventObject().myname toggle = event.GetEventObject().GetValue() if name == 'Check all': self.on_check_all(toggle) return node = anytree.cachedsearch.find(self.script_tree, lambda node: node.id == name) if not get_attribute(node, 'content'): node.content = get_resource(node.id) self.content_text.SetValue(name + "\n\n" + str(node.content)) self.check_boxes(toggle, node) def check_boxes(self, toggle: bool, node: AnyNode): """ Check (toggle = true) or uncheck (toggle = false) boxes corresponding to node while keeping dependencies intact. All ancestors of node are also checked if node is checked, and all children of node are also unchecked if node is unchecked. """ if toggle: while node.depth > 1: self.script_buttons[node.button].SetValue(True) try: self.blocked_urls.remove(node.id) except ValueError: logging.debug("Could not remove %s from blocked urls", node.id) node = node.parent self.script_buttons[node.button].SetValue(True) if node.id[:6] != "script": try: self.blocked_urls.remove(node.id) except ValueError: logging.debug("Could not remove %s from blocked urls", node.id) else: for descendant in node.descendants: self.script_buttons[descendant.button].SetValue(False) self.blocked_urls.append(descendant.id) if node.id[:6] != "script": self.blocked_urls.append(node.id) def on_diff_press(self): """Print diff to terminal.""" after = BeautifulSoup( self.driver.execute_script( "return document.getElementsByTagName('html')[0].innerHTML"), 'html.parser') try: file_stream = open("after.html", "r") before = file_stream.read() file_stream.close() file_stream = open("before.html", "w") file_stream.write(before) file_stream.close() before = BeautifulSoup(before, 'html.parser') except IOError: pass file_stream = open("after.html", "w") file_stream.write(after.prettify()) file_stream.close() os.system(r"git diff --no-index before.html after.html") # os.system(r"diff before.html after.html | sed '/<!--script/,/<\/script>/d'") def on_save(self): """Generate report and save in reports folder.""" if not os.path.exists(PATH + "/reports"): os.mkdir(PATH + "/reports") file_path = PATH + "/reports/" + self.url.split("/")[2] if not os.path.exists(file_path): os.mkdir(file_path) logging.info("Writing script files...") critical = open(file_path + '/critical.txt', 'w') noncritical = open(file_path + '/noncritical.txt', 'w') webalmanac = open(file_path + '/webalmanac.txt', 'w') # labels = open(PATH + "/reports/labels.csv", 'a') for node in PreOrderIter(self.script_tree): if node.is_root or node.id[:6] == 'script': continue checkbox = self.script_buttons[get_attribute(node, 'button')] if checkbox.GetValue(): critical.write(node.id + "\n") else: noncritical.write(node.id + "\n") label = get_attribute(checkbox, 'label') if label and label.GetLabel() != 'critical' and label.GetLabel( ) != 'non-critical': webalmanac.write(node.id + "\n") webalmanac.write(label.GetLabel() + "\n") # if checkbox.GetValue(): # labels.write(str(node.vector.to_list()) + "," + # labels.write(str(node.vector) + "," + # label.GetLabel() + ",critical\n") # else: # labels.write(str(node.vector.to_list()) + "," + # labels.write(str(node.vector) + "," + # label.GetLabel() + ",non-critical\n") critical.close() noncritical.close() webalmanac.close() # labels.close() logging.info("Writing index file...") index = open(file_path + '/index.html', 'w') # pickle.dump(get_resource(self.url + self.suffix) + "\n", index) index.write(get_resource(self.url + self.suffix)) index.close() logging.info("Writing images file...") images = open(file_path + '/images.txt', 'w') for url, dimensions in self.images.items(): images.write(url + "\n") if 'ow' in dimensions and 'oh' in dimensions: images.write("original: %d x %d\n" % (dimensions['ow'], dimensions['oh'])) if 'rw' in dimensions and 'rh' in dimensions: images.write("rendered: %d x %d\n" % (dimensions['rw'], dimensions['rh'])) images.close() self.err_msg.SetForegroundColour((0, 0, 0)) logging.info("Report generated in %s", file_path) # Send report to proxy multipart_form_data = { 'html_content': (open(file_path + '/index.html', 'rb')), 'blocked_URLs': (open(file_path + '/noncritical.txt', 'rb')), 'images': (open(file_path + '/images.txt', 'rb')), 'url': self.url, } response = requests.post("http://" + REMOTE_PROXY_IP + ":9000/JSCleaner/JSAnalyzer.py", files=multipart_form_data) if response.status_code == 200: self.err_msg.SetLabel("Report sent to external proxy") else: self.err_msg.SetLabel( "Report could not be sent - report generated in %s" % file_path) logging.error(response.status_code) logging.error(response.headers) logging.debug(response.text) # Load simplified page from proxy simplified_options = webdriver.ChromeOptions() simplified_options.add_argument('--proxy-server=' + REMOTE_PROXY_IP + ':8083') simplified = webdriver.Chrome(options=simplified_options) simplified.set_window_size(600, 750) simplified.set_window_position(650, 0) simplified.get(self.url + '?JSCleaner') input() simplified.close() def on_choice(self, event): """Handle choiceBox selection.""" choice_box = self.choice_boxes[event.GetEventObject().index] colors = { "": wx.Colour(255, 255, 255, 100), "critical": wx.Colour(255, 0, 0, 100), "non-critical": wx.Colour(0, 255, 0, 100), "translatable": wx.Colour(0, 0, 255, 100) } choice_box.SetBackgroundColour(colors[choice_box.GetValue()]) def parse_log(self, epoch_in_milliseconds): """Return list of scripts requested since epoch_in_milliseconds.""" scripts = [] images = [] log = self.driver.get_log('performance') log = log[bisect.bisect_left([entry['timestamp'] for entry in log], epoch_in_milliseconds ):] log = [json.loads(entry['message'])['message'] for entry in log] def is_script_request(message): if message['method'] == 'Network.requestWillBeSent': if message['params']['type'] == 'Script': return True return False def is_image_request(message): if message['method'] == 'Network.requestWillBeSent': if message['params']['type'] == 'Image': return True return False # def is_script_response(message): # if message['method'] == 'Network.responseReceived': # if 'javascript' in message['params']['response']['mimeType']: # return True # return False def is_data_received(message): if message['method'] == 'Network.dataReceived': return True return False def get_request_info(message): request_id = message['params']['requestId'] request_url = message['params']['request']['url'] initiator = message['params']['initiator'] if initiator['type'] == 'parser': # from index.html as src, need to identify script number somehow... # there are line numbers but are those usable? initiator = initiator['url'] elif initiator['type'] == 'script': # pick last thing in callFrames because first thing doesn't always have URL? # need better understanding # each script has its own ID... if only I could figure out how to use it if initiator['stack']['callFrames']: initiator = initiator['stack']['callFrames'][-1]['url'] return [request_id, request_url, initiator] script_requests = [] # script_responses = [] image_requests = [] data_received = [] for message in log: if is_script_request(message): script_requests.append(message) # elif is_script_response(message): # script_responses.append(message['params']['requestId']) elif is_image_request(message): image_requests.append(message) elif is_data_received(message): data_received.append(message['params']['requestId']) for request in script_requests: request_id, url, initiator = get_request_info(request) if request_id in data_received: content = get_resource(url) scripts.append({ 'url': url, 'parent': initiator, 'content': content }) for request in image_requests: request_id, url, initiator = get_request_info(request) if request_id in data_received: images.append(url) return (scripts, images) def print_scripts(self): """Print script tree.""" print(RenderTree(self.script_tree).by_attr('id')) print("---" * 20) def print_blocked_scripts(self): """Print blocked URLs.""" print('BLOCKED SCRIPTS:') for url in self.blocked_urls: print("\t", url)