def adblock_update(self):
"""Update the adblock block lists.
This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded
host lists and re-reads `~/.config/qutebrowser/blocked-hosts`.
"""
self._read_hosts_file(self._config_hosts_file,
self._config_blocked_hosts)
self._blocked_hosts = set()
self._done_count = 0
download_manager = objreg.get('qtnetwork-download-manager')
for url in config.val.content.host_blocking.lists:
if url.scheme() == 'file':
filename = url.toLocalFile()
if os.path.isdir(filename):
for entry in os.scandir(filename):
if entry.is_file():
self._import_local(entry.path)
else:
self._import_local(filename)
else:
fobj = io.BytesIO()
fobj.name = 'adblock: ' + url.host()
target = downloads.FileObjDownloadTarget(fobj)
download = download_manager.get(url, target=target,
auto_remove=True)
self._in_progress.append(download)
download.finished.connect(
functools.partial(self._on_download_finished, download))
class TestDownloadTarget:
def test_filename(self):
target = downloads.FileDownloadTarget("/foo/bar")
assert target.filename == "/foo/bar"
def test_fileobj(self):
fobj = object()
target = downloads.FileObjDownloadTarget(fobj)
assert target.fileobj is fobj
def test_openfile(self):
target = downloads.OpenFileDownloadTarget()
assert target.cmdline is None
def test_openfile_custom_command(self):
target = downloads.OpenFileDownloadTarget('echo')
assert target.cmdline == 'echo'
@pytest.mark.parametrize('obj', [
downloads.FileDownloadTarget('foobar'),
downloads.FileObjDownloadTarget(None),
downloads.OpenFileDownloadTarget(),
])
def test_class_hierarchy(self, obj):
assert isinstance(obj, downloads._DownloadTarget)
def _fetch_url(self, url):
"""Download the given url and add the file to the collection.
Args:
url: The file to download as QUrl.
"""
if url.scheme() not in ['http', 'https']:
return
# Prevent loading an asset twice
if url in self.loaded_urls:
return
self.loaded_urls.add(url)
log.downloads.debug("loading asset at {}".format(url))
# Using the download manager to download host-blocked urls might crash
# qute, see the comments/discussion on
# https://github.com/qutebrowser/qutebrowser/pull/962#discussion_r40256987
# and https://github.com/qutebrowser/qutebrowser/issues/1053
host_blocker = objreg.get('host-blocker')
if host_blocker.is_blocked(url):
log.downloads.debug("Skipping {}, host-blocked".format(url))
# We still need an empty file in the output, QWebView can be pretty
# picky about displaying a file correctly when not all assets are
# at least referenced in the mhtml file.
self.writer.add_file(urlutils.encoded_url(url), b'')
return
download_manager = objreg.get('qtnetwork-download-manager')
target = downloads.FileObjDownloadTarget(_NoCloseBytesIO())
item = download_manager.get(url, target=target, auto_remove=True)
self.pending_downloads.add((url, item))
item.finished.connect(functools.partial(self._finished, url, item))
item.error.connect(functools.partial(self._error, url, item))
item.cancelled.connect(functools.partial(self._cancelled, url, item))
def adblock_update(self):
"""Update the adblock block lists.
This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded
host lists and re-reads `~/.config/qutebrowser/blocked-hosts`.
"""
self._read_hosts_file(self._config_hosts_file,
self._config_blocked_hosts)
self._blocked_hosts = set()
self._done_count = 0
download_manager = objreg.get('qtnetwork-download-manager')
for url in config.val.content.host_blocking.lists:
if url.scheme() == 'file':
filename = url.toLocalFile()
try:
fileobj = open(filename, 'rb')
except OSError as e:
message.error("adblock: Error while reading {}: {}".format(
filename, e.strerror))
continue
download = FakeDownload(fileobj)
self._in_progress.append(download)
self.on_download_finished(download)
else:
fobj = io.BytesIO()
fobj.name = 'adblock: ' + url.host()
target = downloads.FileObjDownloadTarget(fobj)
download = download_manager.get(url,
target=target,
auto_remove=True)
self._in_progress.append(download)
download.finished.connect(
functools.partial(self.on_download_finished, download))
def download_temp(url: QUrl) -> TempDownload:
"""Download the given URL into a file object.
The download is not saved to disk.
Returns a ``TempDownload`` object, which triggers a ``finished`` signal
when the download has finished::
dl = downloads.download_temp(QUrl("https://www.example.com/"))
dl.finished.connect(functools.partial(on_download_finished, dl))
After the download has finished, its ``successful`` attribute can be
checked to make sure it finished successfully. If so, its contents can be
read by accessing the ``fileobj`` attribute::
def on_download_finished(download: downloads.TempDownload) -> None:
if download.successful:
print(download.fileobj.read())
download.fileobj.close()
"""
fobj = io.BytesIO()
fobj.name = 'temporary: ' + url.host()
target = downloads.FileObjDownloadTarget(fobj)
download_manager = objreg.get('qtnetwork-download-manager')
# cache=False is set as a WORKAROUND for MS Defender thinking we're a trojan
# downloader when caching the hostblock list...
return download_manager.get(url, target=target, auto_remove=True, cache=False)
def download_temp(url: QUrl) -> TempDownload:
"""Download the given URL into a file object.
The download is not saved to disk.
"""
fobj = io.BytesIO()
fobj.name = 'temporary: ' + url.host()
target = downloads.FileObjDownloadTarget(fobj)
download_manager = objreg.get('qtnetwork-download-manager')
return download_manager.get(url, target=target, auto_remove=True)
def adblock_update(self):
"""Update the adblock block lists.
This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded
host lists and re-reads `~/.config/qutebrowser/blocked-hosts`.
"""
self._read_hosts_file(self._config_hosts_file,
self._config_blocked_hosts)
if self._local_hosts_file is None:
raise cmdexc.CommandError("No data storage is configured!")
self._blocked_hosts = set()
self._done_count = 0
urls = config.get('content', 'host-block-lists')
download_manager = objreg.get('qtnetwork-download-manager',
scope='window',
window='last-focused')
if urls is None:
return
for url in urls:
if url.scheme() == 'file':
try:
fileobj = open(url.path(), 'rb')
except OSError as e:
message.error("adblock: Error while reading {}: {}".format(
url.path(), e.strerror))
continue
download = FakeDownload(fileobj)
self._in_progress.append(download)
self.on_download_finished(download)
else:
fobj = io.BytesIO()
fobj.name = 'adblock: ' + url.host()
target = downloads.FileObjDownloadTarget(fobj)
download = download_manager.get(url,
target=target,
auto_remove=True)
self._in_progress.append(download)
download.finished.connect(
functools.partial(self.on_download_finished, download))
def test_fileobj(self):
fobj = object()
target = downloads.FileObjDownloadTarget(fobj)
assert target.fileobj is fobj
