async def cacheFromURL(self, url: URL, name: str) -> Path:
"""
Download a resource and cache it.
"""
cacheDir = self.config.CachedResourcesPath
cacheDir.mkdir(exist_ok=True)
destination = cacheDir / name
if not destination.exists():
with NamedTemporaryFile(dir=str(cacheDir),
delete=False,
suffix=".tmp") as tmp:
path = Path(tmp.name)
try:
await downloadPage(url.asText().encode("utf-8"), tmp)
except BaseException as e:
self._log.failure("Download failed for {url}: {error}",
url=url,
error=e)
try:
path.unlink()
except (OSError, IOError) as e:
self._log.critical(
"Failed to remove temporary file {path}: {error}",
path=path,
error=e)
else:
path.rename(destination)
return destination
async def _httpGET(self,
url: URL,
headers: Headers = emptyHeaders) -> IResponse:
from twisted.internet import reactor
agent = Agent(reactor, pool=self._connectionPool())
return agent.request(b"GET",
url.asText().encode("utf-8"), headers, None)
def redirect(
request: IRequest, location: URL, origin: Optional[str] = None
) -> KleinRenderable:
"""
Perform a redirect.
"""
if origin is not None:
try:
location = location.set(origin, request.uri.decode("utf-8"))
except ValueError:
return badRequestResponse(request, "Invalid origin URI")
log.debug(
"Redirect {source} -> {destination}",
source=request.uri.decode("utf-8"), destination=location.asText(),
)
url = location.asText().encode("utf-8")
request.setHeader(HeaderName.contentType.value, ContentType.html.value)
request.setHeader(HeaderName.location.value, url)
request.setResponseCode(http.FOUND)
return RedirectPage(location=location)
def redirect(request: IRequest,
location: URL,
origin: Optional[str] = None) -> KleinRenderable:
"""
Perform a redirect.
"""
if origin is not None:
try:
location = location.set(origin, request.uri.decode("utf-8"))
except ValueError:
return badRequestResponse(request, "Invalid origin URI")
log.debug(
"Redirect {source} -> {destination}",
source=request.uri.decode("utf-8"),
destination=location.asText(),
)
url = location.asText().encode("utf-8")
request.setHeader(HeaderName.contentType.value, ContentType.html.value)
request.setHeader(HeaderName.location.value, url)
request.setResponseCode(http.FOUND)
return RedirectPage(location)
async def get(
url: hyperlink.URL,
destination: trio.Path,
update_period: float = 0.2,
clock: typing.Callable[[], float] = time.monotonic,
http_application: typing.Optional[typing.Callable[..., typing.Any]] = None,
) -> typing.AsyncIterable[Progress]:
async with httpx.AsyncClient(app=http_application) as client:
async with client.stream("GET", url.asText()) as response:
raw_content_length = response.headers.get("content-length")
if raw_content_length is None:
content_length = None
else:
content_length = int(raw_content_length)
progress = Progress(
downloaded=0,
total=content_length,
first=True,
)
yield progress
last_update = clock()
progress = attr.evolve(progress, first=False)
downloaded = 0
async with (await destination.open("wb")) as file:
async for chunk in response.aiter_raw():
downloaded += len(chunk)
await file.write(chunk) # type: ignore[attr-defined]
if clock() - last_update > update_period:
progress = attr.evolve(progress, downloaded=downloaded)
yield progress
last_update = clock()
if progress.downloaded != downloaded:
progress = attr.evolve(progress, downloaded=downloaded)
yield progress
async def serve(
self,
url: hyperlink.URL,
destination: trio.Path,
) -> None:
self.progress_dialog = qtrio.dialogs.create_progress_dialog()
self.progress_dialog.title = create_title("Fetching")
self.progress_dialog.text = f"Fetching {url}..."
async with self.progress_dialog.manage():
if self.progress_dialog.dialog is None: # pragma: no cover
raise qtrio.InternalError(
"Dialog not assigned while it is being managed.")
# Always show the dialog
self.progress_dialog.dialog.setMinimumDuration(0)
self.progress_dialog_shown_event.set()
start = self.clock()
async for progress in get(
url=url,
destination=destination,
update_period=1 / self.fps,
clock=self.clock,
http_application=self.http_application,
):
if progress.first:
if progress.total is None:
maximum = 0
else:
maximum = progress.total
self.progress_dialog.dialog.setMaximum(maximum)
self.progress_dialog.dialog.setValue(0)
if progress.total is not None:
self.progress_dialog.dialog.setValue(progress.downloaded)
end = self.clock()
self.progress_dialog = None
duration = end - start
if duration == 0:
# define this seems to happen when testing on Windows with an x86 Python
if progress.downloaded > 0:
bytes_per_second = math.inf
else: # pragma: no cover
bytes_per_second = 0
else:
bytes_per_second = progress.downloaded / duration
summary = "\n\n".join([
url.asText(),
os.fspath(destination),
f"Downloaded {progress.downloaded} bytes in {duration:.2f} seconds",
f"{bytes_per_second:.2f} bytes/second",
])
self.message_box = qtrio.dialogs.create_message_box()
self.message_box.title = create_title("Download Summary")
self.message_box.text = summary
await self.message_box.wait(shown_event=self.message_box_shown_event)
self.message_box = None