async def _write_async(self, response: httpx.Response) -> None:
filepath = self._get_filepath(response.url.path)
async with aiofiles.open(filepath, "wb") as f:
async for chunk in response.aiter_bytes(
): # httpx doesn't yet support chunk_size arg
if chunk:
await f.write(chunk)
async def _retrieve_and_write_to_disk(*, response: httpx.Response,
outfile: Path, mode: Literal['ab', 'wb'],
desc: str, local_file_size: int,
remote_file_size: int,
remote_file_hash: Optional[str],
verify_hash: bool,
verify_size: bool) -> None:
hash = hashlib.md5()
# If we're resuming a download, ensure the already-downloaded
# parts of the file are fed into the hash function before
# we continue.
if verify_hash and local_file_size > 0:
async with aiofiles.open(outfile, 'rb') as f:
while True:
data = await f.read(65536)
if not data:
break
hash.update(data)
async with aiofiles.open(outfile, mode=mode) as f:
with tqdm(desc=desc,
initial=local_file_size,
total=remote_file_size,
unit='B',
unit_scale=True,
unit_divisor=1024,
leave=False) as progress:
num_bytes_downloaded = response.num_bytes_downloaded
# TODO Add timeout handling here, too.
async for chunk in response.aiter_bytes():
await f.write(chunk)
progress.update(response.num_bytes_downloaded -
num_bytes_downloaded)
num_bytes_downloaded = (response.num_bytes_downloaded)
if verify_hash:
hash.update(chunk)
if verify_hash and remote_file_hash is not None:
assert hash.hexdigest() == remote_file_hash
# Check the file was completely downloaded.
if verify_size:
await f.flush()
local_file_size = outfile.stat().st_size
if not local_file_size == remote_file_size:
raise RuntimeError(
f'Server claimed file size would be {remote_file_size} '
f'bytes, but downloaded {local_file_size} byes.')
async def read_json_resp(resp: httpx.Response) -> Dict[str, Any]:
data = b""
async for chunk in resp.aiter_bytes():
data += chunk
return orjson.loads(data)
