def _is_blocked(
self,
request_url: QUrl,
first_party_url: Optional[QUrl] = None,
resource_type: Optional[interceptor.ResourceType] = None,
) -> bool:
"""Check whether the given request is blocked."""
if not self.enabled:
# Do nothing if `content.blocking.method` is not set to enable the
# use of this adblocking module.
return False
if (
first_party_url is None
or not first_party_url.isValid()
or first_party_url.scheme() == "file"
):
# FIXME: It seems that when `first_party_url` is None, every URL
# I try is blocked. This may have been a result of me incorrectly
# using the upstream library, or an upstream bug. For now we don't
# block any request with `first_party_url=None`.
return False
qtutils.ensure_valid(request_url)
if not config.get("content.blocking.enabled", url=first_party_url):
# Do nothing if adblocking is disabled for this site.
return False
result = self._engine.check_network_urls(
request_url.toString(),
first_party_url.toString(),
_resource_type_to_string(resource_type),
)
if not result.matched:
return False
elif result.exception is not None and not result.important:
# Exception is not `None` when the blocker matched on an exception
# rule. Effectively this means that there was a match, but the
# request should not be blocked.
#
# An `important` match means that exceptions should not apply and
# no further checking is necessary--the request should be blocked.
logger.debug(
"Excepting %s from being blocked by %s because of %s",
request_url.toDisplayString(),
result.filter,
result.exception,
)
return False
elif blockutils.is_whitelisted_url(request_url):
logger.debug(
"Request to %s is whitelisted, thus not blocked",
request_url.toDisplayString(),
)
return False
return True
def data_for_url(url: QUrl) -> Tuple[str, bytes]:
"""Get the data to show for the given URL.
Args:
url: The QUrl to show.
Return:
A (mimetype, data) tuple.
"""
norm_url = url.adjusted(
QUrl.NormalizePathSegments | # type: ignore[arg-type]
QUrl.StripTrailingSlash)
if norm_url != url:
raise Redirect(norm_url)
path = url.path()
host = url.host()
query = url.query()
# A url like "qute:foo" is split as "scheme:path", not "scheme:host".
log.misc.debug("url: {}, path: {}, host {}".format(
url.toDisplayString(), path, host))
if not path or not host:
new_url = QUrl()
new_url.setScheme('qute')
# When path is absent, e.g. qute://help (with no trailing slash)
if host:
new_url.setHost(host)
# When host is absent, e.g. qute:help
else:
new_url.setHost(path)
new_url.setPath('/')
if query:
new_url.setQuery(query)
if new_url.host(): # path was a valid host
raise Redirect(new_url)
try:
handler = _HANDLERS[host]
except KeyError:
raise NotFoundError("No handler found for {}".format(
url.toDisplayString()))
try:
mimetype, data = handler(url)
except OSError as e:
raise SchemeOSError(e)
assert mimetype is not None, url
if mimetype == 'text/html' and isinstance(data, str):
# We let handlers return HTML as text
data = data.encode('utf-8', errors='xmlcharrefreplace')
assert isinstance(data, bytes)
return mimetype, data
class HistoryEntry:
"""A single entry in the web history.
Attributes:
atime: The time the page was accessed.
url: The URL which was accessed as QUrl.
url_string: The URL which was accessed as string.
"""
def __init__(self, atime, url):
self.atime = float(atime)
self.url = QUrl(url)
self.url_string = url
def __repr__(self):
return utils.get_repr(self,
constructor=True,
atime=self.atime,
url=self.url.toDisplayString())
def __str__(self):
return '{} {}'.format(int(self.atime), self.url_string)
@classmethod
def from_str(cls, s):
"""Get a history based on a 'TIME URL' string."""
return cls(*s.split(' ', maxsplit=1))
def _on_history_trigger(self):
try:
self._widget.page()
except RuntimeError:
# Looks like this slot can be triggered on destroyed tabs:
# https://crashes.qutebrowser.org/view/3abffbed (Qt 5.9.1)
# wrapped C/C++ object of type WebEngineView has been deleted
log.misc.debug("Ignoring history trigger for destroyed tab")
return
url = self.url()
requested_url = self.url(requested=True)
# Don't save the title if it's generated from the URL
title = self.title()
title_url = QUrl(url)
title_url.setScheme('')
if title == title_url.toDisplayString(QUrl.RemoveScheme).strip('/'):
title = ""
# Don't add history entry if the URL is invalid anyways
if not url.isValid():
log.misc.debug("Ignoring invalid URL being added to history")
return
self.add_history_item.emit(url, requested_url, title)
def wait_for_load_finished_url(self, url, *, timeout=None,
load_status='success'):
"""Wait until a URL has finished loading."""
__tracebackhide__ = (lambda e: e.errisinstance(
testprocess.WaitForTimeout))
if timeout is None:
if 'CI' in os.environ:
timeout = 15000
else:
timeout = 5000
# We really need the same representation that the webview uses in its
# __repr__
qurl = QUrl(url)
if not qurl.isValid():
raise ValueError("Invalid URL {}: {}".format(url,
qurl.errorString()))
url = utils.elide(qurl.toDisplayString(QUrl.EncodeUnicode), 100)
assert url
pattern = re.compile(
r"(load status for <qutebrowser\.browser\..* "
r"tab_id=\d+ url='{url}/?'>: LoadStatus\.{load_status}|fetch: "
r"PyQt5\.QtCore\.QUrl\('{url}'\) -> .*)".format(
load_status=re.escape(load_status), url=re.escape(url)))
try:
self.wait_for(message=pattern, timeout=timeout)
except testprocess.WaitForTimeout:
raise testprocess.WaitForTimeout("Timed out while waiting for {} "
"to be loaded".format(url))
def acceptNavigationRequest(self, url: QUrl,
typ: QWebEnginePage.NavigationType,
is_main_frame: bool):
"""Override acceptNavigationRequest to handle clicked links.
Setting linkDelegationPolicy to DelegateAllLinks and using a slot bound
to linkClicked won't work correctly, because when in a frameset, we
have no idea in which frame the link should be opened.
Checks if it should open it in a tab (middle-click or control) or not,
and then conditionally opens the URL. Opening it in a new tab/window
is handled in the slot connected to link_clicked.
"""
target = self._tabdata.combined_target()
log.webview.debug("navigation request: url {}, type {}, "
"target {}, is_main_frame {}".format(
url.toDisplayString(),
debug.qenum_key(QWebEnginePage, typ), target,
is_main_frame))
if typ != QWebEnginePage.NavigationTypeLinkClicked:
return True
self.link_clicked.emit(url)
return url.isValid() and target == usertypes.ClickTarget.normal
def acceptNavigationRequest(self,
url: QUrl,
typ: QWebEnginePage.NavigationType,
is_main_frame: bool):
"""Override acceptNavigationRequest to handle clicked links.
Setting linkDelegationPolicy to DelegateAllLinks and using a slot bound
to linkClicked won't work correctly, because when in a frameset, we
have no idea in which frame the link should be opened.
Checks if it should open it in a tab (middle-click or control) or not,
and then conditionally opens the URL. Opening it in a new tab/window
is handled in the slot connected to link_clicked.
"""
target = self._tabdata.combined_target()
log.webview.debug("navigation request: url {}, type {}, "
"target {}, is_main_frame {}".format(
url.toDisplayString(),
debug.qenum_key(QWebEnginePage, typ),
target, is_main_frame))
if typ != QWebEnginePage.NavigationTypeLinkClicked:
return True
self.link_clicked.emit(url)
return url.isValid() and target == usertypes.ClickTarget.normal
class HistoryEntry:
"""A single entry in the web history.
Attributes:
atime: The time the page was accessed.
url: The URL which was accessed as QUrl.
url_string: The URL which was accessed as string.
hidden: If True, don't save this entry to disk
"""
def __init__(self, atime, url, title, hidden=False):
self.atime = float(atime)
self.url = QUrl(url)
self.url_string = url
self.title = title
self.hidden = hidden
def __repr__(self):
return utils.get_repr(self,
constructor=True,
atime=self.atime,
url=self.url.toDisplayString(),
title=self.title,
hidden=self.hidden)
def __str__(self):
return '{} {} {}'.format(int(self.atime), self.url_string, self.title)
def _on_history_trigger(self):
try:
self._widget.page()
except RuntimeError:
# Looks like this slot can be triggered on destroyed tabs:
# https://crashes.qutebrowser.org/view/3abffbed (Qt 5.9.1)
# wrapped C/C++ object of type WebEngineView has been deleted
log.misc.debug("Ignoring history trigger for destroyed tab")
return
url = self.url()
requested_url = self.url(requested=True)
# Don't save the title if it's generated from the URL
title = self.title()
title_url = QUrl(url)
title_url.setScheme('')
if title == title_url.toDisplayString(QUrl.RemoveScheme).strip('/'):
title = ""
# Don't add history entry if the URL is invalid anyways
if not url.isValid():
log.misc.debug("Ignoring invalid URL being added to history")
return
self.add_history_item.emit(url, requested_url, title)
def _get_hints_arg(self, *, origin_url: QUrl,
icon: QImage) -> Dict[str, Any]:
"""Get the hints argument for present()."""
origin_url_str = origin_url.toDisplayString()
hints: Dict[str, Any] = {
# Include the origin in case the user wants to do different things
# with different origin's notifications.
"x-qutebrowser-origin": origin_url_str,
"desktop-entry": "org.qutebrowser.qutebrowser",
}
is_useful_origin = self._should_include_origin(origin_url)
if self._capabilities.kde_origin_name and is_useful_origin:
hints["x-kde-origin-name"] = origin_url_str
if icon.isNull():
filename = 'icons/qutebrowser-64x64.png'
icon = QImage.fromData(resources.read_file_binary(filename))
key = self._quirks.icon_key or "image-data"
data = self._convert_image(icon)
if data is not None:
hints[key] = data
return hints
class HistoryEntry:
"""A single entry in the web history.
Attributes:
atime: The time the page was accessed.
url: The URL which was accessed as QUrl.
url_string: The URL which was accessed as string.
hidden: If True, don't save this entry to disk
"""
def __init__(self, atime, url, title, hidden=False):
self.atime = float(atime)
self.url = QUrl(url)
self.url_string = url
self.title = title
self.hidden = hidden
def __repr__(self):
return utils.get_repr(self, constructor=True, atime=self.atime,
url=self.url.toDisplayString(), title=self.title,
hidden=self.hidden)
def __str__(self):
return '{} {} {}'.format(int(self.atime), self.url_string, self.title)
def _format_body(self, body: str, origin_url: QUrl) -> str:
"""Format the body according to the server capabilities.
If the server doesn't support x-kde-origin-name, we include the origin URL as a
prefix. If possible, we hyperlink it.
For both prefix and body, we'll need to HTML escape it if the server supports
body markup.
"""
urlstr = origin_url.toDisplayString()
is_useful_origin = self._should_include_origin(origin_url)
if self._capabilities.kde_origin_name or not is_useful_origin:
prefix = None
elif self._capabilities.body_markup and self._capabilities.body_hyperlinks:
href = html.escape(
origin_url.toString(
QUrl.FullyEncoded) # type: ignore[arg-type]
)
text = html.escape(urlstr, quote=False)
prefix = f'<a href="{href}">{text}</a>'
elif self._capabilities.body_markup:
prefix = html.escape(urlstr, quote=False)
else:
prefix = urlstr
if self._capabilities.body_markup:
body = html.escape(body, quote=False)
if prefix is None:
return body
return prefix + '\n\n' + body
def test_resource_url():
"""Test resource_url() which can be used from templates."""
data = jinja.render('test2.html')
print(data)
url = QUrl(data)
assert url.isValid()
assert url.toDisplayString() == 'qute://resource/utils/testfile'
def createRequest(self, op, req, outgoing_data):
"""Return a new QNetworkReply object.
Args:
op: Operation op
req: const QNetworkRequest & req
outgoing_data: QIODevice * outgoingData
Return:
A QNetworkReply.
"""
if proxymod.application_factory is not None:
proxy_error = proxymod.application_factory.get_error()
if proxy_error is not None:
return networkreply.ErrorNetworkReply(
req, proxy_error, QNetworkReply.UnknownProxyError, self)
if not req.url().isValid():
log.network.debug("Ignoring invalid requested URL: {}".format(
req.url().errorString()))
return networkreply.ErrorNetworkReply(
req, "Invalid request URL", QNetworkReply.HostNotFoundError,
self)
for header, value in shared.custom_headers(url=req.url()):
req.setRawHeader(header, value)
tab = self._get_tab()
current_url = QUrl()
if tab is not None:
try:
current_url = tab.url()
except RuntimeError:
# We could be in the middle of the webpage shutdown here.
pass
request = interceptors.Request(first_party_url=current_url,
request_url=req.url())
interceptors.run(request)
if request.is_blocked:
return networkreply.ErrorNetworkReply(
req, HOSTBLOCK_ERROR_STRING, QNetworkReply.ContentAccessDenied,
self)
if 'log-requests' in objects.debug_flags:
operation = debug.qenum_key(QNetworkAccessManager, op)
operation = operation.replace('Operation', '').upper()
log.network.debug("{} {}, first-party {}".format(
operation,
req.url().toDisplayString(), current_url.toDisplayString()))
scheme = req.url().scheme()
if scheme in self._scheme_handlers:
result = self._scheme_handlers[scheme](req, op, current_url)
if result is not None:
result.setParent(self)
return result
self.set_referer(req, current_url)
return super().createRequest(op, req, outgoing_data)
class HistoryEntry:
"""A single entry in the web history.
Attributes:
atime: The time the page was accessed.
url: The URL which was accessed as QUrl.
url_string: The URL which was accessed as string.
"""
def __init__(self, atime, url):
self.atime = float(atime)
self.url = QUrl(url)
self.url_string = url
def __repr__(self):
return utils.get_repr(self, constructor=True, atime=self.atime, url=self.url.toDisplayString())
def __str__(self):
return "{} {}".format(int(self.atime), self.url_string)
@classmethod
def from_str(cls, s):
"""Get a history based on a 'TIME URL' string."""
return cls(*s.split(" ", maxsplit=1))
def update_source(self):
if self.settings.contains('protofile'):
source_url = QUrl(self.settings.value('protofile'))
source = source_url.toDisplayString(QUrl.RemoveScheme)
if os.path.isfile(source):
self.source_url = source_url
self.set_source(source)
return not self.source_url.isEmpty()
def _handle_errorpage(self, info, errpage):
"""Display an error page if needed.
Loosly based on Helpviewer/HelpBrowserWV.py from eric5
(line 260 @ 5d937eb378dd)
Args:
info: The QWebPage.ErrorPageExtensionOption instance.
errpage: The QWebPage.ErrorPageExtensionReturn instance, where the
error page will get written to.
Return:
False if no error page should be displayed, True otherwise.
"""
ignored_errors = [
(QWebPage.QtNetwork, QNetworkReply.OperationCanceledError),
(QWebPage.WebKit, 203), # "Loading is handled by the media engine"
]
errpage.baseUrl = info.url
urlstr = info.url.toDisplayString()
if (info.domain, info.error) == (QWebPage.QtNetwork,
QNetworkReply.ProtocolUnknownError):
# For some reason, we get a segfault when we use
# QDesktopServices::openUrl with info.url directly - however it
# works when we construct a copy of it.
url = QUrl(info.url)
msg = "Open external application for {}-link?\nURL: {}".format(
url.scheme(), url.toDisplayString())
message.confirm_async(
self._win_id, msg,
functools.partial(QDesktopServices.openUrl, url))
return True
elif (info.domain, info.error) in ignored_errors:
log.webview.debug("Ignored error on {}: {} (error domain: {}, "
"error code: {})".format(
urlstr, info.errorString, info.domain,
info.error))
return False
else:
error_str = info.errorString
if error_str == networkmanager.HOSTBLOCK_ERROR_STRING:
error_str = "Request blocked by host blocker."
# we don't set error_occured in this case.
else:
self._ignore_load_started = True
self.error_occured = True
log.webview.error("Error while loading {}: {}".format(
urlstr, error_str))
log.webview.debug("Error domain: {}, error code: {}".format(
info.domain, info.error))
title = "Error loading page: {}".format(urlstr)
template = jinja.env.get_template('error.html')
html = template.render( # pylint: disable=maybe-no-member
title=title, url=urlstr, error=error_str, icon='')
errpage.content = html.encode('utf-8')
errpage.encoding = 'utf-8'
return True
def addEngineFromForm(self, res, view):
"""
Public method to add a new search engine from a form.
@param res result of the JavaScript run on by
WebBrowserView.__addSearchEngine()
@type dict or None
@param view reference to the web browser view
@type WebBrowserView
"""
if not res:
return
method = res["method"]
actionUrl = QUrl(res["action"])
inputName = res["inputName"]
if method != "get":
E5MessageBox.warning(
self, self.tr("Method not supported"),
self.tr("""{0} method is not supported.""").format(
method.upper()))
return
if actionUrl.isRelative():
actionUrl = view.url().resolved(actionUrl)
searchUrlQuery = QUrlQuery(actionUrl)
searchUrlQuery.addQueryItem(inputName, "{searchTerms}")
inputFields = res["inputs"]
for inputField in inputFields:
name = inputField[0]
value = inputField[1]
if not name or name == inputName or not value:
continue
searchUrlQuery.addQueryItem(name, value)
engineName, ok = QInputDialog.getText(
view, self.tr("Engine name"),
self.tr("Enter a name for the engine"), QLineEdit.Normal)
if not ok:
return
actionUrl.setQuery(searchUrlQuery)
from .OpenSearchEngine import OpenSearchEngine
engine = OpenSearchEngine()
engine.setName(engineName)
engine.setDescription(engineName)
engine.setSearchUrlTemplate(
actionUrl.toDisplayString(QUrl.FullyDecoded))
engine.setImage(view.icon().pixmap(16, 16).toImage())
self.__addEngineByEngine(engine)
def preset_cmd_text(self, url: QUrl, context: HintContext) -> None:
"""Preset a commandline text based on a hint URL."""
urlstr = url.toDisplayString(QUrl.FullyEncoded) # type: ignore
args = context.get_args(urlstr)
text = ' '.join(args)
if text[0] not in modeparsers.STARTCHARS:
raise HintingError("Invalid command text '{}'.".format(text))
cmd = objreg.get('status-command', scope='window', window=self._win_id)
cmd.set_cmd_text(text)
def safe_display_string(qurl: QUrl) -> str:
"""Get a IDN-homograph phishing safe form of the given QUrl.
If we're dealing with a Punycode-encoded URL, this prepends the hostname in
its encoded form, to make sure those URLs are distinguishable.
See https://github.com/qutebrowser/qutebrowser/issues/2547
and https://bugreports.qt.io/browse/QTBUG-60365
"""
ensure_valid(qurl)
host = qurl.host(QUrl.FullyEncoded)
assert '..' not in host, qurl # https://bugreports.qt.io/browse/QTBUG-60364
for part in host.split('.'):
url_host = qurl.host(QUrl.FullyDecoded)
if part.startswith('xn--') and host != url_host:
return '({}) {}'.format(host, qurl.toDisplayString())
return qurl.toDisplayString()
def _handle_errorpage(self, info, errpage):
"""Display an error page if needed.
Loosly based on Helpviewer/HelpBrowserWV.py from eric5
(line 260 @ 5d937eb378dd)
Args:
info: The QWebPage.ErrorPageExtensionOption instance.
errpage: The QWebPage.ErrorPageExtensionReturn instance, where the
error page will get written to.
Return:
False if no error page should be displayed, True otherwise.
"""
ignored_errors = [
(QWebPage.QtNetwork, QNetworkReply.OperationCanceledError),
(QWebPage.WebKit, 203), # "Loading is handled by the media engine"
]
errpage.baseUrl = info.url
urlstr = info.url.toDisplayString()
if (info.domain, info.error) == (QWebPage.QtNetwork,
QNetworkReply.ProtocolUnknownError):
# For some reason, we get a segfault when we use
# QDesktopServices::openUrl with info.url directly - however it
# works when we construct a copy of it.
url = QUrl(info.url)
msg = "Open external application for {}-link?\nURL: {}".format(
url.scheme(), url.toDisplayString())
message.confirm_async(
self._win_id, msg,
functools.partial(QDesktopServices.openUrl, url))
return True
elif (info.domain, info.error) in ignored_errors:
log.webview.debug("Ignored error on {}: {} (error domain: {}, "
"error code: {})".format(urlstr,
info.errorString,
info.domain,
info.error))
return False
else:
log.webview.error("Error while loading {}: {}".format(
urlstr, info.errorString))
log.webview.debug("Error domain: {}, error code: {}".format(
info.domain, info.error))
title = "Error loading page: {}".format(urlstr)
template = jinja.env.get_template('error.html')
html = template.render( # pylint: disable=maybe-no-member
title=title,
url=urlstr,
error=info.errorString,
icon='')
errpage.content = html.encode('utf-8')
errpage.encoding = 'utf-8'
return True
def _on_history_trigger(self):
url = self.url()
requested_url = self.url(requested=True)
# Don't save the title if it's generated from the URL
title = self.title()
title_url = QUrl(url)
title_url.setScheme('')
if title == title_url.toDisplayString(QUrl.RemoveScheme).strip('/'):
title = ""
self.add_history_item.emit(url, requested_url, title)
def _on_history_trigger(self):
url = self.url()
requested_url = self.url(requested=True)
# Don't save the title if it's generated from the URL
title = self.title()
title_url = QUrl(url)
title_url.setScheme('')
if title == title_url.toDisplayString(QUrl.RemoveScheme).strip('/'):
title = ""
self.add_history_item.emit(url, requested_url, title)
def invalid_url_error(url: QUrl, action: str) -> None:
"""Display an error message for a URL.
Args:
action: The action which was interrupted by the error.
"""
if url.isValid():
raise ValueError("Calling invalid_url_error with valid URL {}".format(
url.toDisplayString()))
errstring = get_errstring(url,
"Trying to {} with invalid URL".format(action))
message.error(errstring)
def data_for_url(url: QUrl):
"""Get the data to show for the given URL.
Args:
url: The QUrl to show.
Return:
A (mimetype, data) tuple.
"""
norm_url = url.adjusted(QUrl.NormalizePathSegments
| QUrl.StripTrailingSlash)
if norm_url != url:
raise Redirect(norm_url)
path = url.path()
host = url.host()
query = urlutils.query_string(url)
# A url like "luminos:foo" is split as "scheme:path", not "scheme:host".
if not path or not host:
new_url = QUrl()
new_url.setScheme("luminos")
# When path is absent, e.g. luminos://help (with no trailing slash)
if host:
new_url.setHost(host)
# When host is absent, e.g. luminos:help
else:
new_url.setHost(path)
new_url.setPath("/")
if query:
new_url.setQuery(query)
if new_url.host(): # path was a valid host
raise Redirect(new_url)
try:
handler = _HANDLERS[host]
except KeyError:
raise NotFoundError("No handler found for {}".format(
url.toDisplayString()))
try:
mimetype, data = handler(url)
except OSError as e:
raise SchemeOSError(e)
assert mimetype is not None, url
if mimetype == "text/html" and isinstance(data, str):
# We let handlers return HTML as text
data = data.encode("utf-8", errors="xmlcharrefreplace")
return mimetype, data
def host_tuple(url: QUrl) -> HostTupleType:
"""Get a (scheme, host, port) tuple from a QUrl.
This is suitable to identify a connection, e.g. for SSL errors.
"""
ensure_valid(url)
scheme, host, port = url.scheme(), url.host(), url.port()
assert scheme
if not host:
raise ValueError("Got URL {} without host.".format(
url.toDisplayString()))
if port == -1:
port_mapping = {
'http': 80,
'https': 443,
'ftp': 21,
}
try:
port = port_mapping[scheme]
except KeyError:
raise ValueError("Got URL {} with unknown port.".format(
url.toDisplayString()))
return scheme, host, port
def get_image(self, size):
if self.source_setting_changed():
self.update_source()
image = QImage(size.width(), size.height(), QImage.Format_ARGB32)
painter = QPainter(image)
if self.graph is None:
if self.settings.contains('protofile'):
url = QUrl(self.settings.value('protofile'))
self.set_source(url.toDisplayString(QUrl.RemoveScheme))
else:
image.fill(QColor(200, 200, 200))
return image
self.renderer.load(QByteArray(self.graph))
self.renderer.render(painter)
return image
def _on_history_trigger(self):
url = self.url()
requested_url = self.url(requested=True)
# Don't save the title if it's generated from the URL
title = self.title()
title_url = QUrl(url)
title_url.setScheme('')
if title == title_url.toDisplayString(QUrl.RemoveScheme).strip('/'):
title = ""
# Don't add history entry if the URL is invalid anyways
if not url.isValid():
log.misc.debug("Ignoring invalid URL being added to history")
return
self.add_history_item.emit(url, requested_url, title)
def _on_history_trigger(self):
url = self.url()
requested_url = self.url(requested=True)
# Don't save the title if it's generated from the URL
title = self.title()
title_url = QUrl(url)
title_url.setScheme('')
if title == title_url.toDisplayString(QUrl.RemoveScheme).strip('/'):
title = ""
# Don't add history entry if the URL is invalid anyways
if not url.isValid():
log.misc.debug("Ignoring invalid URL being added to history")
return
self.add_history_item.emit(url, requested_url, title)
def wait_for_load_finished_url(self,
url,
*,
timeout=None,
load_status='success',
after=None):
"""Wait until a URL has finished loading."""
__tracebackhide__ = (
lambda e: e.errisinstance(testprocess.WaitForTimeout))
if timeout is None:
if 'CI' in os.environ:
timeout = 15000
else:
timeout = 5000
qurl = QUrl(url)
if not qurl.isValid():
raise ValueError("Invalid URL {}: {}".format(
url, qurl.errorString()))
if (qurl == QUrl('about:blank')
and not qtutils.version_check('5.10', compiled=False)):
# For some reason, we don't get a LoadStatus.success for
# about:blank sometimes.
# However, if we do this for Qt 5.10, we get general testsuite
# instability as site loads get reported with about:blank...
pattern = "Changing title for idx * to 'about:blank'"
else:
# We really need the same representation that the webview uses in
# its __repr__
url = utils.elide(qurl.toDisplayString(QUrl.EncodeUnicode), 100)
assert url
pattern = re.compile(
r"(load status for <qutebrowser\.browser\..* "
r"tab_id=\d+ url='{url}/?'>: LoadStatus\.{load_status}|fetch: "
r"PyQt5\.QtCore\.QUrl\('{url}'\) -> .*)".format(
load_status=re.escape(load_status), url=re.escape(url)))
try:
self.wait_for(message=pattern, timeout=timeout, after=after)
except testprocess.WaitForTimeout:
raise testprocess.WaitForTimeout("Timed out while waiting for {} "
"to be loaded".format(url))
def set_hover_url(self, link):
"""Setter to be used as a Qt slot.
Saves old shown URL in self._old_url and restores it later if a link is
"un-hovered" when it gets called with empty parameters.
Args:
link: The link which was hovered (string)
"""
if link:
qurl = QUrl(link)
if qurl.isValid():
self._hover_url = qurl.toDisplayString()
else:
self._hover_url = link
else:
self._hover_url = None
self._update_url()
def acceptNavigationRequest(self, url: QUrl,
typ: QWebEnginePage.NavigationType,
is_main_frame: bool):
"""Override acceptNavigationRequest to handle clicked links.
This only show an error on invalid links - everything else is handled
in createWindow.
"""
log.webview.debug("navigation request: url {}, type {}, is_main_frame "
"{}".format(url.toDisplayString(),
debug.qenum_key(QWebEnginePage, typ),
is_main_frame))
if (typ == QWebEnginePage.NavigationTypeLinkClicked
and not url.isValid()):
msg = urlutils.get_errstring(url, "Invalid link clicked")
message.error(msg)
return False
return True
def acceptNavigationRequest(self,
url: QUrl,
typ: QWebEnginePage.NavigationType,
is_main_frame: bool):
"""Override acceptNavigationRequest to handle clicked links.
This only show an error on invalid links - everything else is handled
in createWindow.
"""
log.webview.debug("navigation request: url {}, type {}, is_main_frame "
"{}".format(url.toDisplayString(),
debug.qenum_key(QWebEnginePage, typ),
is_main_frame))
if (typ == QWebEnginePage.NavigationTypeLinkClicked and
not url.isValid()):
msg = urlutils.get_errstring(url, "Invalid link clicked")
message.error(msg)
return False
return True
def set_hover_url(self, link, _title, _text):
"""Setter to be used as a Qt slot.
Saves old shown URL in self._old_url and restores it later if a link is
"un-hovered" when it gets called with empty parameters.
Args:
link: The link which was hovered (string)
_title: The title of the hovered link (string)
_text: The text of the hovered link (string)
"""
if link:
qurl = QUrl(link)
if qurl.isValid():
self._hover_url = qurl.toDisplayString()
else:
self._hover_url = link
else:
self._hover_url = None
self._update_url()
def wait_for_load_finished_url(self, url, *, timeout=None,
load_status='success', after=None):
"""Wait until a URL has finished loading."""
__tracebackhide__ = (lambda e: e.errisinstance(
testprocess.WaitForTimeout))
if timeout is None:
if 'CI' in os.environ:
timeout = 15000
else:
timeout = 5000
qurl = QUrl(url)
if not qurl.isValid():
raise ValueError("Invalid URL {}: {}".format(url,
qurl.errorString()))
if (qurl == QUrl('about:blank') and
not qtutils.version_check('5.10', compiled=False)):
# For some reason, we don't get a LoadStatus.success for
# about:blank sometimes.
# However, if we do this for Qt 5.10, we get general testsuite
# instability as site loads get reported with about:blank...
pattern = "Changing title for idx * to 'about:blank'"
else:
# We really need the same representation that the webview uses in
# its __repr__
url = utils.elide(qurl.toDisplayString(QUrl.EncodeUnicode), 100)
assert url
pattern = re.compile(
r"(load status for <qutebrowser\.browser\..* "
r"tab_id=\d+ url='{url}/?'>: LoadStatus\.{load_status}|fetch: "
r"PyQt5\.QtCore\.QUrl\('{url}'\) -> .*)".format(
load_status=re.escape(load_status), url=re.escape(url)))
try:
self.wait_for(message=pattern, timeout=timeout, after=after)
except testprocess.WaitForTimeout:
raise testprocess.WaitForTimeout("Timed out while waiting for {} "
"to be loaded".format(url))
def data_for_custom_scheme(url: QUrl):
""""""
scheme = url.scheme()
try:
handler = _CUSTOM_HANDLERS[scheme]
except KeyError:
raise NotFoundError("No handler found for scheme {}".format(
url.toDisplayString()))
try:
mimetype, data = handler(url)
except OSError as e:
raise SchemeOSError(e)
assert mimetype is not None, url
if mimetype == "text/html" and isinstance(data, str):
# We let handlers return HTML as text
data = data.encode("utf-8", errors="xmlcharrefreplace")
return mimetype, data
def update_for_url(self, url: QUrl) -> typing.Set[str]:
"""Update settings customized for the given tab.
Return:
A set of settings which actually changed.
"""
qtutils.ensure_valid(url)
changed_settings = set()
for values in config.instance:
if not values.opt.supports_pattern:
continue
value = values.get_for_url(url, fallback=False)
changed = self._update_setting(values.opt.name, value)
if changed:
log.config.debug("Changed for {}: {} = {}".format(
url.toDisplayString(), values.opt.name, value))
changed_settings.add(values.opt.name)
return changed_settings
def update_for_url(self, url: QUrl) -> typing.Set[str]:
"""Update settings customized for the given tab.
Return:
A set of settings which actually changed.
"""
qtutils.ensure_valid(url)
changed_settings = set()
for values in config.instance:
if not values.opt.supports_pattern:
continue
value = values.get_for_url(url, fallback=False)
changed = self._update_setting(values.opt.name, value)
if changed:
log.config.debug("Changed for {}: {} = {}".format(
url.toDisplayString(), values.opt.name, value))
changed_settings.add(values.opt.name)
return changed_settings
def set_hover_url(self, link, _title, _text):
"""Setter to be used as a Qt slot.
Saves old shown URL in self._old_url and restores it later if a link is
"un-hovered" when it gets called with empty parameters.
Args:
link: The link which was hovered (string)
_title: The title of the hovered link (string)
_text: The text of the hovered link (string)
"""
if link:
# We assume that `link` is always be given in a form that generates
# a valid QUrl. If this proves to be wrong, we should probably
# check and fall back to the text version otherwise.
qurl = QUrl(link)
assert qurl.isValid(), link
self._hover_url = qurl.toDisplayString()
else:
self._hover_url = None
self._update_url()
class HistoryEntry:
"""A single entry in the web history.
Attributes:
atime: The time the page was accessed.
url: The URL which was accessed as QUrl.
url_string: The URL which was accessed as string.
"""
def __init__(self, atime, url):
self.atime = float(atime)
self.url = QUrl(url)
self.url_string = url
def __repr__(self):
return utils.get_repr(self, constructor=True, atime=self.atime,
url=self.url.toDisplayString())
def __str__(self):
return '{} {}'.format(int(self.atime), self.url_string)
class HistoryEntry:
"""A single entry in the web history.
Attributes:
atime: The time the page was accessed.
url: The URL which was accessed as QUrl.
url_string: The URL which was accessed as string.
"""
def __init__(self, atime, url):
self.atime = float(atime)
self.url = QUrl(url)
self.url_string = url
def __repr__(self):
return utils.get_repr(self,
constructor=True,
atime=self.atime,
url=self.url.toDisplayString())
def __str__(self):
return '{} {}'.format(int(self.atime), self.url_string)
def __addFeed(self, button):
"""
Private slot to add a RSS feed.
@param button reference to the feed button
@type QPushButton
"""
urlString = button.feed[1]
url = QUrl(urlString)
if url.isRelative():
url = self.__browser.url().resolved(url)
urlString = url.toDisplayString(QUrl.FullyDecoded)
if not url.isValid():
return
if button.feed[0]:
title = button.feed[0]
else:
title = self.__browser.url().host()
from WebBrowser.WebBrowserWindow import WebBrowserWindow
feedsManager = WebBrowserWindow.feedsManager()
if feedsManager.addFeed(urlString, title, self.__browser.icon()):
if WebBrowserWindow.notificationsEnabled():
WebBrowserWindow.showNotification(
UI.PixmapCache.getPixmap("rss48.png"),
self.tr("Add RSS Feed"),
self.tr("""The feed was added successfully."""))
else:
E5MessageBox.information(
self, self.tr("Add RSS Feed"),
self.tr("""The feed was added successfully."""))
else:
E5MessageBox.warning(
self, self.tr("Add RSS Feed"),
self.tr("""The feed was already added before."""))
self.close()
def createRequest(self, op, req, outgoing_data):
"""Return a new QNetworkReply object.
Args:
op: Operation op
req: const QNetworkRequest & req
outgoing_data: QIODevice * outgoingData
Return:
A QNetworkReply.
"""
proxy_factory = objreg.get('proxy-factory', None)
if proxy_factory is not None:
proxy_error = proxy_factory.get_error()
if proxy_error is not None:
return networkreply.ErrorNetworkReply(
req, proxy_error, QNetworkReply.UnknownProxyError,
self)
for header, value in shared.custom_headers(url=req.url()):
req.setRawHeader(header, value)
# There are some scenarios where we can't figure out current_url:
# - There's a generic NetworkManager, e.g. for downloads
# - The download was in a tab which is now closed.
current_url = QUrl()
if self._tab_id is not None:
assert self._win_id is not None
try:
tab = objreg.get('tab', scope='tab', window=self._win_id,
tab=self._tab_id)
current_url = tab.url()
except (KeyError, RuntimeError):
# https://github.com/qutebrowser/qutebrowser/issues/889
# Catching RuntimeError because we could be in the middle of
# the webpage shutdown here.
current_url = QUrl()
request = interceptors.Request(first_party_url=current_url,
request_url=req.url())
interceptors.run(request)
if request.is_blocked:
return networkreply.ErrorNetworkReply(
req, HOSTBLOCK_ERROR_STRING, QNetworkReply.ContentAccessDenied,
self)
if 'log-requests' in self._args.debug_flags:
operation = debug.qenum_key(QNetworkAccessManager, op)
operation = operation.replace('Operation', '').upper()
log.webview.debug("{} {}, first-party {}".format(
operation,
req.url().toDisplayString(),
current_url.toDisplayString()))
scheme = req.url().scheme()
if scheme in self._scheme_handlers:
result = self._scheme_handlers[scheme](req, op, current_url)
if result is not None:
result.setParent(self)
return result
self.set_referer(req, current_url)
return super().createRequest(op, req, outgoing_data)
def _handle_errorpage(self, info, errpage):
"""Display an error page if needed.
Loosely based on Helpviewer/HelpBrowserWV.py from eric5
(line 260 @ 5d937eb378dd)
Args:
info: The QWebPage.ErrorPageExtensionOption instance.
errpage: The QWebPage.ErrorPageExtensionReturn instance, where the
error page will get written to.
Return:
False if no error page should be displayed, True otherwise.
"""
ignored_errors = [
(QWebPage.QtNetwork, QNetworkReply.OperationCanceledError),
# "Loading is handled by the media engine"
(QWebPage.WebKit, 203),
# "Frame load interrupted by policy change"
(QWebPage.WebKit, 102),
]
errpage.baseUrl = info.url
urlstr = info.url.toDisplayString()
if (info.domain, info.error) == (QWebPage.QtNetwork,
QNetworkReply.ProtocolUnknownError):
# For some reason, we get a segfault when we use
# QDesktopServices::openUrl with info.url directly - however it
# works when we construct a copy of it.
url = QUrl(info.url)
scheme = url.scheme()
message.confirm_async(
title="Open external application for {}-link?".format(scheme),
text="URL: <b>{}</b>".format(html.escape(
url.toDisplayString())),
yes_action=functools.partial(QDesktopServices.openUrl, url))
return True
elif (info.domain, info.error) in ignored_errors:
log.webview.debug("Ignored error on {}: {} (error domain: {}, "
"error code: {})".format(urlstr,
info.errorString,
info.domain,
info.error))
return False
else:
error_str = info.errorString
if error_str == networkmanager.HOSTBLOCK_ERROR_STRING:
# We don't set error_occurred in this case.
error_str = "Request blocked by host blocker."
main_frame = info.frame.page().mainFrame()
if info.frame != main_frame:
# Content in an iframe -> Hide the frame so it doesn't use
# any space. We can't hide the frame's documentElement
# directly though.
for elem in main_frame.documentElement().findAll('iframe'):
if QUrl(elem.attribute('src')) == info.url:
elem.setAttribute('style', 'display: none')
return False
else:
self._ignore_load_started = True
self.error_occurred = True
log.webview.error("Error while loading {}: {}".format(
urlstr, error_str))
log.webview.debug("Error domain: {}, error code: {}".format(
info.domain, info.error))
title = "Error loading page: {}".format(urlstr)
error_html = jinja.render('error.html',
title=title,
url=urlstr,
error=error_str)
errpage.content = error_html.encode('utf-8')
errpage.encoding = 'utf-8'
return True
def _handle_errorpage(self, info, errpage):
"""Display an error page if needed.
Loosely based on Helpviewer/HelpBrowserWV.py from eric5
(line 260 @ 5d937eb378dd)
Args:
info: The QWebPage.ErrorPageExtensionOption instance.
errpage: The QWebPage.ErrorPageExtensionReturn instance, where the
error page will get written to.
Return:
False if no error page should be displayed, True otherwise.
"""
ignored_errors = [
(QWebPage.QtNetwork, QNetworkReply.OperationCanceledError),
# "Loading is handled by the media engine"
(QWebPage.WebKit, 203),
# "Frame load interrupted by policy change"
(QWebPage.WebKit, 102),
]
errpage.baseUrl = info.url
urlstr = info.url.toDisplayString()
if (info.domain, info.error) == (QWebPage.QtNetwork,
QNetworkReply.ProtocolUnknownError):
# For some reason, we get a segfault when we use
# QDesktopServices::openUrl with info.url directly - however it
# works when we construct a copy of it.
url = QUrl(info.url)
msg = "Open external application for {}-link?\nURL: {}".format(
url.scheme(), url.toDisplayString())
message.confirm_async(
self._win_id, msg,
functools.partial(QDesktopServices.openUrl, url))
return True
elif (info.domain, info.error) in ignored_errors:
log.webview.debug("Ignored error on {}: {} (error domain: {}, "
"error code: {})".format(
urlstr, info.errorString, info.domain,
info.error))
return False
else:
error_str = info.errorString
if error_str == networkmanager.HOSTBLOCK_ERROR_STRING:
# We don't set error_occurred in this case.
error_str = "Request blocked by host blocker."
main_frame = info.frame.page().mainFrame()
if info.frame != main_frame:
# Content in an iframe -> Hide the frame so it doesn't use
# any space. We can't hide the frame's documentElement
# directly though.
for elem in main_frame.documentElement().findAll('iframe'):
if QUrl(elem.attribute('src')) == info.url:
elem.setAttribute('style', 'display: none')
return False
else:
self._ignore_load_started = True
self.error_occurred = True
log.webview.error("Error while loading {}: {}".format(
urlstr, error_str))
log.webview.debug("Error domain: {}, error code: {}".format(
info.domain, info.error))
title = "Error loading page: {}".format(urlstr)
html = jinja.render(
'error.html',
title=title, url=urlstr, error=error_str, icon='')
errpage.content = html.encode('utf-8')
errpage.encoding = 'utf-8'
return True
def _on_before_load_started(self, url: QUrl) -> None:
"""Adjust the title if we are going to visit a URL soon."""
qtutils.ensure_valid(url)
url_string = url.toDisplayString()
log.webview.debug("Going to start loading: {}".format(url_string))
self.title_changed.emit(url_string)
def _on_url_changed(self, url: QUrl) -> None:
"""Update title when URL has changed and no title is available."""
if url.isValid() and not self.title():
self.title_changed.emit(url.toDisplayString())
self.url_changed.emit(url)
def _on_before_load_started(self, url: QUrl) -> None:
"""Adjust the title if we are going to visit a URL soon."""
qtutils.ensure_valid(url)
url_string = url.toDisplayString()
log.webview.debug("Going to start loading: {}".format(url_string))
self.title_changed.emit(url_string)
def _on_url_changed(self, url: QUrl) -> None:
"""Update title when URL has changed and no title is available."""
if url.isValid() and not self.title():
self.title_changed.emit(url.toDisplayString())
self.url_changed.emit(url)
