def __init__(self, instance, parent=None):
super(DiskCacheDir, self).__init__(parent)
self.setCacheDirectory(
expanduser("~/.eilat/caches/cache{prefix}".format_map(
get_options()['sites'][instance])))
self.setMaximumCacheSize(1024 * 1024 * 128)
def proxy_options():
""" Extracts the proxy information from YAML user settings. If any is
empty, proxy will be disabled. If the fields do not exist (not checked)
the app will fail hard.
"""
options = get_options()['proxy']
return (options['host'], options['port'])
def proxy_options():
""" Extracts the proxy information from YAML user settings. If any is
empty, proxy will be disabled. If the fields do not exist (not checked)
the app will fail hard.
"""
options = get_options()['proxy']
return (options['host'], options['port'])
def extract_instance(url):
""" Given a site, give the key to a map that decides if cookies are
allowed, if only some sites will not be blocked, etc.
"""
host = None if url is None else tldextract.extract(url).domain
options_sites = get_options()['sites']
if host in options_sites.keys():
return host
else:
return 'general'
def extract_instance(url):
""" Given a site, give the key to a map that decides if cookies are
allowed, if only some sites will not be blocked, etc.
"""
host = None if url is None else tldextract.extract(url).domain
options_sites = get_options()['sites']
if host in options_sites.keys():
return host
else:
return 'general'
def on_link_click(qurl):
""" Callback for connection. Reads the 'paste' attribute
to know if a middle click requested to open on a new tab.
BRW01 BRW12 VID01 CB03
"""
# required for 'open in new tab if not in this instance'
qurl = do_redirect(qurl) # BRW12
# 'play' and 'save' should only happen from inside the webkit and
# if the user started the action; handle here, not in 'navigate'
if 'play' in self.attr:
print("PLAYING")
GLOBAL_VID_PLAY.play(qurl) # VID01
self.attr.clear('play')
return
if 'save' in self.attr:
clipboard(qurl) # CB02
self.attr.clear('save')
return
instance = extract_instance(qurl.toString())
target_prefix = get_options()['sites'][instance]['prefix']
# if the prefixes don't match, we're requesting a new instance
# TAB04 TAB05
if 'paste' in self.attr or self.attr.prefix != target_prefix:
mainwin().add_tab(qurl,
scripting=('open_scripted' in self.attr))
self.attr.clear('paste')
else:
self.navigate(qurl) # BRW01
if 'open_scripted' in self.attr:
self.attr.clear('open_scripted')
def create_request(self, operation, request, data):
""" Reimplemented to intercept requests. Stops blacklisted requests
CFG02
"""
request.setAttribute(
QNetworkRequest.HttpPipeliningAllowedAttribute, True)
request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
QNetworkRequest.PreferCache)
qurl = request.url()
url = qurl.toString()
info = {'notify': self.show_detail,
'message': None,
'detail': None,
'qurl': qurl,
'color': Fore.GREEN}
# if keeping a log of the POST data, do it before anything else
if operation == QNetworkAccessManager.PostOperation:
notify_post(data, url)
# stop here if the request is local enough as for not
# requiring further scrutiny
if is_local(qurl) and not is_font(qurl):
info['message'] = "LOCAL"
return self.__pass(operation, request, data, info=info)
# make optional the loading of webfonts
if is_font(qurl):
if self.load_webfonts:
return self.__pass(operation, request, data)
else:
info['message'] = "TRIMMED WEBFONT"
return self.__block("about:blank", info=info)
# If the request is going to be intercepted a custom request is
# built and returned after optionally reporting the reason
whitelist = get_options()['sites'][self.__instance]['host_whitelist']
info['color'] = Fore.RED
for (stop_case, description, show) in [
# it may be an un-dns'ed request; careful here
(is_numerical(qurl.host()), "NUMERICAL", True),
# whitelist exists, and the requested URL is not in it
(non_whitelisted(whitelist, qurl),
"NON WHITELISTED", self.show_detail)
]:
if stop_case:
info['notify'] = show
info['message'] = description
return self.__block("about:blank", info=info)
# It's not a local request; it should have a proper URL structure
# then. 'domain' and 'suffix' must be non-None (and non-empty).
(subdomain, domain, suffix) = tldextract.extract(url)
subdomain = subdomain if subdomain != '' else None
# FIXME this entire block (and maybe the one above) feels half-baked
for (stop_case, description, detail, show) in [
# if 'domain' or 'suffix' are not valid, stop;
# should never happen (even though it does - some providers
# don't have a proper 'domain' according to tldextract
(domain == '' or suffix == '',
"SHOULD NOT HAPPEN", " {}|{}|{} ".format(subdomain,
domain,
suffix), True),
# site is on a whitelist, but this instance does not use
# whitelists; this means it belong to another instance,
# assuming exclusivity
# TODO 'domain + suffix' is a maybe bad generalization
(whitelist is None and # CFG02
domain + '.' + suffix in get_options()['all_whitelists'],
"WHITELIST^C FILTER", "{} || {} || {} ".format(
subdomain,
domain,
suffix), self.show_detail)
]:
if stop_case:
info['message'] = description
info['detail'] = detail
return self.__block(encode_blocked(description, url),
info=info)
return self.__pass(operation, request, data)
def __init__(self, instance, parent=None):
super(InterceptNAM, self).__init__(parent)
print("INIT InterceptNAM")
self.__instance = instance
self.prefix = get_options()['sites'][instance]['prefix']
self.show_detail = False
self.load_webfonts = False
# reference needed to save in shutdown
self.cookie_jar = CookieJar(parent=self,
options=get_options()['sites'][instance])
self.setCookieJar(self.cookie_jar) # COO02
self.setCache(DiskCacheDir(instance, parent=self))
def reply_complete(reply):
""" Prints when a request completes, handles the filter that
chooses between successful and filtered requests
Replies only - if the request doesn't have to go
through the network, it will not be reported here
AB01
"""
qurl = reply.url()
s_url = qurl.toString()
status = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
if is_local(qurl) or status is None:
return
# 'status' will never be None from here on
# used only once
color_status = Fore.GREEN if reply.attribute(
QNetworkRequest.SourceIsFromCacheAttribute) else Fore.RED
# was it a filtered reply?
# used only once
color = Fore.BLUE if 200 <= status < 400 else Fore.RED
origin = reply.request().originatingObject()
if origin:
# used only once
not_same = not compare_host(
origin.requestedUrl().host(), qurl.host())
else:
print("NO ORIGINATING OBJECT", s_url)
return
in_iframe = origin != origin.page().mainFrame()
# this is the response to the original .load(), not a
# resource loaded by the page itself
if origin.requestedUrl() == qurl and not in_iframe:
host = sub("^www.", "", qurl.host())
path = qurl.path().rstrip("/ ")
if ( # AB01
(host not in DO_NOT_STORE) and
(not qurl.hasQuery()) and
len(path.split('/')) < 4 and
200 <= status < 400):
database().store_navigation(host, path, self.prefix)
if not_same or in_iframe:
info = {
'color_status': color_status,
'status': status,
'resource': qurl,
'source': origin.requestedUrl(),
'main_color': color,
'iframe': in_iframe}
show_info(info)
# finished carries QNetworkReply
self.finished.connect(reply_complete)
def handle_ssl_error(reply, errors):
""" Callback to connect to when a SSL error happens
This ignores the error before reporting it; that means all
"issuer certificate could not be found" and similar will be
accepted but reported. Until a better way to handle is
implemented, keep an eye on the console when counting on SSL.
This can and will create security issues.
"""
reply.ignoreSslErrors()
notify("[S]")
show_labeled("SSL",
reply.url(),
detail="/".join([
k.certificate().toText() +
k.errorString()
for k in errors]),
color=Fore.RED)
self.sslErrors.connect(handle_ssl_error)
def navigate(self, request=None):
""" Open the url on this tab. If 'url' is already a QUrl
(if it comes from a href click), just send it. Otherwise,
it comes either from the address bar or the PRIMARY
clipboard through a keyboard shortcut.
Check if the "url" is actually one, partial or otherwise;
if it's not, construct a web search.
BRW01
"""
if isinstance(request, QUrl):
qurl = request
# 'navigate' was connected to a QLineEdit to extract its text
elif isinstance(request, QLineEdit):
url = request.text()
qurl = fix_url(url) # BRW11
else:
raise RuntimeError("Navigating to non-navigable")
# if the qurl does not trigger an URL in SHORTENERS or REDIRECTORS,
# this will be a no-op
qurl = do_redirect(qurl) # BRW12
if self.attr.prefix is None:
# this is the first navigation on this tab/webkit; replace
# the Network Access Manager CFG02
instance = extract_instance(qurl.toString())
self.attr.set_prefix(get_options()['sites'][instance]['prefix'])
if self.attr.prefix is None:
raise RuntimeError(
"prefix failed to be set... 'options' is broken")
# strictly speaking, this should emit from Attributes.set_prefix
self.set_prefix.emit(self.attr.prefix)
if self.attr.prefix == "":
print("GENERAL")
else:
print("INSTANCE: {}".format(self.attr.prefix))
if not has_manager(self.attr.prefix):
register_manager(self.attr.prefix,
InterceptNAM(instance,
parent=None))
if not has_manager(self.attr.prefix):
raise RuntimeError("prefix manager not registered...")
self.page().setNetworkAccessManager(get_manager(self.attr.prefix))
print("{}>>>\t\t{}\n>>> NAVIGATE {}{}".format(
Fore.CYAN,
datetime.datetime.now().isoformat(),
qurl.toString(),
Fore.RESET))
self.setFocus()
self.load_requested.emit(qurl.toString())
self.load(qurl)
