def __init__(self, instance, parent=None): super(DiskCacheDir, self).__init__(parent) self.setCacheDirectory( expanduser("~/.eilat/caches/cache{prefix}".format_map( get_options()['sites'][instance]))) self.setMaximumCacheSize(1024 * 1024 * 128)
def proxy_options(): """ Extracts the proxy information from YAML user settings. If any is empty, proxy will be disabled. If the fields do not exist (not checked) the app will fail hard. """ options = get_options()['proxy'] return (options['host'], options['port'])
def extract_instance(url): """ Given a site, give the key to a map that decides if cookies are allowed, if only some sites will not be blocked, etc. """ host = None if url is None else tldextract.extract(url).domain options_sites = get_options()['sites'] if host in options_sites.keys(): return host else: return 'general'
def on_link_click(qurl): """ Callback for connection. Reads the 'paste' attribute to know if a middle click requested to open on a new tab. BRW01 BRW12 VID01 CB03 """ # required for 'open in new tab if not in this instance' qurl = do_redirect(qurl) # BRW12 # 'play' and 'save' should only happen from inside the webkit and # if the user started the action; handle here, not in 'navigate' if 'play' in self.attr: print("PLAYING") GLOBAL_VID_PLAY.play(qurl) # VID01 self.attr.clear('play') return if 'save' in self.attr: clipboard(qurl) # CB02 self.attr.clear('save') return instance = extract_instance(qurl.toString()) target_prefix = get_options()['sites'][instance]['prefix'] # if the prefixes don't match, we're requesting a new instance # TAB04 TAB05 if 'paste' in self.attr or self.attr.prefix != target_prefix: mainwin().add_tab(qurl, scripting=('open_scripted' in self.attr)) self.attr.clear('paste') else: self.navigate(qurl) # BRW01 if 'open_scripted' in self.attr: self.attr.clear('open_scripted')
def create_request(self, operation, request, data): """ Reimplemented to intercept requests. Stops blacklisted requests CFG02 """ request.setAttribute( QNetworkRequest.HttpPipeliningAllowedAttribute, True) request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.PreferCache) qurl = request.url() url = qurl.toString() info = {'notify': self.show_detail, 'message': None, 'detail': None, 'qurl': qurl, 'color': Fore.GREEN} # if keeping a log of the POST data, do it before anything else if operation == QNetworkAccessManager.PostOperation: notify_post(data, url) # stop here if the request is local enough as for not # requiring further scrutiny if is_local(qurl) and not is_font(qurl): info['message'] = "LOCAL" return self.__pass(operation, request, data, info=info) # make optional the loading of webfonts if is_font(qurl): if self.load_webfonts: return self.__pass(operation, request, data) else: info['message'] = "TRIMMED WEBFONT" return self.__block("about:blank", info=info) # If the request is going to be intercepted a custom request is # built and returned after optionally reporting the reason whitelist = get_options()['sites'][self.__instance]['host_whitelist'] info['color'] = Fore.RED for (stop_case, description, show) in [ # it may be an un-dns'ed request; careful here (is_numerical(qurl.host()), "NUMERICAL", True), # whitelist exists, and the requested URL is not in it (non_whitelisted(whitelist, qurl), "NON WHITELISTED", self.show_detail) ]: if stop_case: info['notify'] = show info['message'] = description return self.__block("about:blank", info=info) # It's not a local request; it should have a proper URL structure # then. 'domain' and 'suffix' must be non-None (and non-empty). (subdomain, domain, suffix) = tldextract.extract(url) subdomain = subdomain if subdomain != '' else None # FIXME this entire block (and maybe the one above) feels half-baked for (stop_case, description, detail, show) in [ # if 'domain' or 'suffix' are not valid, stop; # should never happen (even though it does - some providers # don't have a proper 'domain' according to tldextract (domain == '' or suffix == '', "SHOULD NOT HAPPEN", " {}|{}|{} ".format(subdomain, domain, suffix), True), # site is on a whitelist, but this instance does not use # whitelists; this means it belong to another instance, # assuming exclusivity # TODO 'domain + suffix' is a maybe bad generalization (whitelist is None and # CFG02 domain + '.' + suffix in get_options()['all_whitelists'], "WHITELIST^C FILTER", "{} || {} || {} ".format( subdomain, domain, suffix), self.show_detail) ]: if stop_case: info['message'] = description info['detail'] = detail return self.__block(encode_blocked(description, url), info=info) return self.__pass(operation, request, data)
def __init__(self, instance, parent=None): super(InterceptNAM, self).__init__(parent) print("INIT InterceptNAM") self.__instance = instance self.prefix = get_options()['sites'][instance]['prefix'] self.show_detail = False self.load_webfonts = False # reference needed to save in shutdown self.cookie_jar = CookieJar(parent=self, options=get_options()['sites'][instance]) self.setCookieJar(self.cookie_jar) # COO02 self.setCache(DiskCacheDir(instance, parent=self)) def reply_complete(reply): """ Prints when a request completes, handles the filter that chooses between successful and filtered requests Replies only - if the request doesn't have to go through the network, it will not be reported here AB01 """ qurl = reply.url() s_url = qurl.toString() status = reply.attribute( QNetworkRequest.HttpStatusCodeAttribute) if is_local(qurl) or status is None: return # 'status' will never be None from here on # used only once color_status = Fore.GREEN if reply.attribute( QNetworkRequest.SourceIsFromCacheAttribute) else Fore.RED # was it a filtered reply? # used only once color = Fore.BLUE if 200 <= status < 400 else Fore.RED origin = reply.request().originatingObject() if origin: # used only once not_same = not compare_host( origin.requestedUrl().host(), qurl.host()) else: print("NO ORIGINATING OBJECT", s_url) return in_iframe = origin != origin.page().mainFrame() # this is the response to the original .load(), not a # resource loaded by the page itself if origin.requestedUrl() == qurl and not in_iframe: host = sub("^www.", "", qurl.host()) path = qurl.path().rstrip("/ ") if ( # AB01 (host not in DO_NOT_STORE) and (not qurl.hasQuery()) and len(path.split('/')) < 4 and 200 <= status < 400): database().store_navigation(host, path, self.prefix) if not_same or in_iframe: info = { 'color_status': color_status, 'status': status, 'resource': qurl, 'source': origin.requestedUrl(), 'main_color': color, 'iframe': in_iframe} show_info(info) # finished carries QNetworkReply self.finished.connect(reply_complete) def handle_ssl_error(reply, errors): """ Callback to connect to when a SSL error happens This ignores the error before reporting it; that means all "issuer certificate could not be found" and similar will be accepted but reported. Until a better way to handle is implemented, keep an eye on the console when counting on SSL. This can and will create security issues. """ reply.ignoreSslErrors() notify("[S]") show_labeled("SSL", reply.url(), detail="/".join([ k.certificate().toText() + k.errorString() for k in errors]), color=Fore.RED) self.sslErrors.connect(handle_ssl_error)
def navigate(self, request=None): """ Open the url on this tab. If 'url' is already a QUrl (if it comes from a href click), just send it. Otherwise, it comes either from the address bar or the PRIMARY clipboard through a keyboard shortcut. Check if the "url" is actually one, partial or otherwise; if it's not, construct a web search. BRW01 """ if isinstance(request, QUrl): qurl = request # 'navigate' was connected to a QLineEdit to extract its text elif isinstance(request, QLineEdit): url = request.text() qurl = fix_url(url) # BRW11 else: raise RuntimeError("Navigating to non-navigable") # if the qurl does not trigger an URL in SHORTENERS or REDIRECTORS, # this will be a no-op qurl = do_redirect(qurl) # BRW12 if self.attr.prefix is None: # this is the first navigation on this tab/webkit; replace # the Network Access Manager CFG02 instance = extract_instance(qurl.toString()) self.attr.set_prefix(get_options()['sites'][instance]['prefix']) if self.attr.prefix is None: raise RuntimeError( "prefix failed to be set... 'options' is broken") # strictly speaking, this should emit from Attributes.set_prefix self.set_prefix.emit(self.attr.prefix) if self.attr.prefix == "": print("GENERAL") else: print("INSTANCE: {}".format(self.attr.prefix)) if not has_manager(self.attr.prefix): register_manager(self.attr.prefix, InterceptNAM(instance, parent=None)) if not has_manager(self.attr.prefix): raise RuntimeError("prefix manager not registered...") self.page().setNetworkAccessManager(get_manager(self.attr.prefix)) print("{}>>>\t\t{}\n>>> NAVIGATE {}{}".format( Fore.CYAN, datetime.datetime.now().isoformat(), qurl.toString(), Fore.RESET)) self.setFocus() self.load_requested.emit(qurl.toString()) self.load(qurl)