async def cacheFromURL(self, url: URL, name: str) -> Path: """ Download a resource and cache it. """ cacheDir = self.config.CachedResourcesPath cacheDir.mkdir(exist_ok=True) destination = cacheDir / name if not destination.exists(): with NamedTemporaryFile(dir=str(cacheDir), delete=False, suffix=".tmp") as tmp: path = Path(tmp.name) try: await downloadPage(url.asText().encode("utf-8"), tmp) except BaseException as e: self._log.failure("Download failed for {url}: {error}", url=url, error=e) try: path.unlink() except (OSError, IOError) as e: self._log.critical( "Failed to remove temporary file {path}: {error}", path=path, error=e) else: path.rename(destination) return destination
async def _httpGET(self, url: URL, headers: Headers = emptyHeaders) -> IResponse: from twisted.internet import reactor agent = Agent(reactor, pool=self._connectionPool()) return agent.request(b"GET", url.asText().encode("utf-8"), headers, None)
def redirect( request: IRequest, location: URL, origin: Optional[str] = None ) -> KleinRenderable: """ Perform a redirect. """ if origin is not None: try: location = location.set(origin, request.uri.decode("utf-8")) except ValueError: return badRequestResponse(request, "Invalid origin URI") log.debug( "Redirect {source} -> {destination}", source=request.uri.decode("utf-8"), destination=location.asText(), ) url = location.asText().encode("utf-8") request.setHeader(HeaderName.contentType.value, ContentType.html.value) request.setHeader(HeaderName.location.value, url) request.setResponseCode(http.FOUND) return RedirectPage(location=location)
def redirect(request: IRequest, location: URL, origin: Optional[str] = None) -> KleinRenderable: """ Perform a redirect. """ if origin is not None: try: location = location.set(origin, request.uri.decode("utf-8")) except ValueError: return badRequestResponse(request, "Invalid origin URI") log.debug( "Redirect {source} -> {destination}", source=request.uri.decode("utf-8"), destination=location.asText(), ) url = location.asText().encode("utf-8") request.setHeader(HeaderName.contentType.value, ContentType.html.value) request.setHeader(HeaderName.location.value, url) request.setResponseCode(http.FOUND) return RedirectPage(location)
async def get( url: hyperlink.URL, destination: trio.Path, update_period: float = 0.2, clock: typing.Callable[[], float] = time.monotonic, http_application: typing.Optional[typing.Callable[..., typing.Any]] = None, ) -> typing.AsyncIterable[Progress]: async with httpx.AsyncClient(app=http_application) as client: async with client.stream("GET", url.asText()) as response: raw_content_length = response.headers.get("content-length") if raw_content_length is None: content_length = None else: content_length = int(raw_content_length) progress = Progress( downloaded=0, total=content_length, first=True, ) yield progress last_update = clock() progress = attr.evolve(progress, first=False) downloaded = 0 async with (await destination.open("wb")) as file: async for chunk in response.aiter_raw(): downloaded += len(chunk) await file.write(chunk) # type: ignore[attr-defined] if clock() - last_update > update_period: progress = attr.evolve(progress, downloaded=downloaded) yield progress last_update = clock() if progress.downloaded != downloaded: progress = attr.evolve(progress, downloaded=downloaded) yield progress
async def serve( self, url: hyperlink.URL, destination: trio.Path, ) -> None: self.progress_dialog = qtrio.dialogs.create_progress_dialog() self.progress_dialog.title = create_title("Fetching") self.progress_dialog.text = f"Fetching {url}..." async with self.progress_dialog.manage(): if self.progress_dialog.dialog is None: # pragma: no cover raise qtrio.InternalError( "Dialog not assigned while it is being managed.") # Always show the dialog self.progress_dialog.dialog.setMinimumDuration(0) self.progress_dialog_shown_event.set() start = self.clock() async for progress in get( url=url, destination=destination, update_period=1 / self.fps, clock=self.clock, http_application=self.http_application, ): if progress.first: if progress.total is None: maximum = 0 else: maximum = progress.total self.progress_dialog.dialog.setMaximum(maximum) self.progress_dialog.dialog.setValue(0) if progress.total is not None: self.progress_dialog.dialog.setValue(progress.downloaded) end = self.clock() self.progress_dialog = None duration = end - start if duration == 0: # define this seems to happen when testing on Windows with an x86 Python if progress.downloaded > 0: bytes_per_second = math.inf else: # pragma: no cover bytes_per_second = 0 else: bytes_per_second = progress.downloaded / duration summary = "\n\n".join([ url.asText(), os.fspath(destination), f"Downloaded {progress.downloaded} bytes in {duration:.2f} seconds", f"{bytes_per_second:.2f} bytes/second", ]) self.message_box = qtrio.dialogs.create_message_box() self.message_box.title = create_title("Download Summary") self.message_box.text = summary await self.message_box.wait(shown_event=self.message_box_shown_event) self.message_box = None