def adblock_update(self): """Update the adblock block lists. This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded host lists and re-reads `~/.config/qutebrowser/blocked-hosts`. """ self._read_hosts_file(self._config_hosts_file, self._config_blocked_hosts) self._blocked_hosts = set() self._done_count = 0 download_manager = objreg.get('qtnetwork-download-manager') for url in config.val.content.host_blocking.lists: if url.scheme() == 'file': filename = url.toLocalFile() if os.path.isdir(filename): for entry in os.scandir(filename): if entry.is_file(): self._import_local(entry.path) else: self._import_local(filename) else: fobj = io.BytesIO() fobj.name = 'adblock: ' + url.host() target = downloads.FileObjDownloadTarget(fobj) download = download_manager.get(url, target=target, auto_remove=True) self._in_progress.append(download) download.finished.connect( functools.partial(self._on_download_finished, download))
class TestDownloadTarget: def test_filename(self): target = downloads.FileDownloadTarget("/foo/bar") assert target.filename == "/foo/bar" def test_fileobj(self): fobj = object() target = downloads.FileObjDownloadTarget(fobj) assert target.fileobj is fobj def test_openfile(self): target = downloads.OpenFileDownloadTarget() assert target.cmdline is None def test_openfile_custom_command(self): target = downloads.OpenFileDownloadTarget('echo') assert target.cmdline == 'echo' @pytest.mark.parametrize('obj', [ downloads.FileDownloadTarget('foobar'), downloads.FileObjDownloadTarget(None), downloads.OpenFileDownloadTarget(), ]) def test_class_hierarchy(self, obj): assert isinstance(obj, downloads._DownloadTarget)
def _fetch_url(self, url): """Download the given url and add the file to the collection. Args: url: The file to download as QUrl. """ if url.scheme() not in ['http', 'https']: return # Prevent loading an asset twice if url in self.loaded_urls: return self.loaded_urls.add(url) log.downloads.debug("loading asset at {}".format(url)) # Using the download manager to download host-blocked urls might crash # qute, see the comments/discussion on # https://github.com/qutebrowser/qutebrowser/pull/962#discussion_r40256987 # and https://github.com/qutebrowser/qutebrowser/issues/1053 host_blocker = objreg.get('host-blocker') if host_blocker.is_blocked(url): log.downloads.debug("Skipping {}, host-blocked".format(url)) # We still need an empty file in the output, QWebView can be pretty # picky about displaying a file correctly when not all assets are # at least referenced in the mhtml file. self.writer.add_file(urlutils.encoded_url(url), b'') return download_manager = objreg.get('qtnetwork-download-manager') target = downloads.FileObjDownloadTarget(_NoCloseBytesIO()) item = download_manager.get(url, target=target, auto_remove=True) self.pending_downloads.add((url, item)) item.finished.connect(functools.partial(self._finished, url, item)) item.error.connect(functools.partial(self._error, url, item)) item.cancelled.connect(functools.partial(self._cancelled, url, item))
def adblock_update(self): """Update the adblock block lists. This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded host lists and re-reads `~/.config/qutebrowser/blocked-hosts`. """ self._read_hosts_file(self._config_hosts_file, self._config_blocked_hosts) self._blocked_hosts = set() self._done_count = 0 download_manager = objreg.get('qtnetwork-download-manager') for url in config.val.content.host_blocking.lists: if url.scheme() == 'file': filename = url.toLocalFile() try: fileobj = open(filename, 'rb') except OSError as e: message.error("adblock: Error while reading {}: {}".format( filename, e.strerror)) continue download = FakeDownload(fileobj) self._in_progress.append(download) self.on_download_finished(download) else: fobj = io.BytesIO() fobj.name = 'adblock: ' + url.host() target = downloads.FileObjDownloadTarget(fobj) download = download_manager.get(url, target=target, auto_remove=True) self._in_progress.append(download) download.finished.connect( functools.partial(self.on_download_finished, download))
def download_temp(url: QUrl) -> TempDownload: """Download the given URL into a file object. The download is not saved to disk. Returns a ``TempDownload`` object, which triggers a ``finished`` signal when the download has finished:: dl = downloads.download_temp(QUrl("https://www.example.com/")) dl.finished.connect(functools.partial(on_download_finished, dl)) After the download has finished, its ``successful`` attribute can be checked to make sure it finished successfully. If so, its contents can be read by accessing the ``fileobj`` attribute:: def on_download_finished(download: downloads.TempDownload) -> None: if download.successful: print(download.fileobj.read()) download.fileobj.close() """ fobj = io.BytesIO() fobj.name = 'temporary: ' + url.host() target = downloads.FileObjDownloadTarget(fobj) download_manager = objreg.get('qtnetwork-download-manager') # cache=False is set as a WORKAROUND for MS Defender thinking we're a trojan # downloader when caching the hostblock list... return download_manager.get(url, target=target, auto_remove=True, cache=False)
def download_temp(url: QUrl) -> TempDownload: """Download the given URL into a file object. The download is not saved to disk. """ fobj = io.BytesIO() fobj.name = 'temporary: ' + url.host() target = downloads.FileObjDownloadTarget(fobj) download_manager = objreg.get('qtnetwork-download-manager') return download_manager.get(url, target=target, auto_remove=True)
def adblock_update(self): """Update the adblock block lists. This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded host lists and re-reads `~/.config/qutebrowser/blocked-hosts`. """ self._read_hosts_file(self._config_hosts_file, self._config_blocked_hosts) if self._local_hosts_file is None: raise cmdexc.CommandError("No data storage is configured!") self._blocked_hosts = set() self._done_count = 0 urls = config.get('content', 'host-block-lists') download_manager = objreg.get('qtnetwork-download-manager', scope='window', window='last-focused') if urls is None: return for url in urls: if url.scheme() == 'file': try: fileobj = open(url.path(), 'rb') except OSError as e: message.error("adblock: Error while reading {}: {}".format( url.path(), e.strerror)) continue download = FakeDownload(fileobj) self._in_progress.append(download) self.on_download_finished(download) else: fobj = io.BytesIO() fobj.name = 'adblock: ' + url.host() target = downloads.FileObjDownloadTarget(fobj) download = download_manager.get(url, target=target, auto_remove=True) self._in_progress.append(download) download.finished.connect( functools.partial(self.on_download_finished, download))
def test_fileobj(self): fobj = object() target = downloads.FileObjDownloadTarget(fobj) assert target.fileobj is fobj