def _cdsapi_download_with_timeout(self, url, size, target):
"""Copied from cdsapi.api, see below."""
from cdsapi.api import bytes_to_string, time, requests
if target is None:
target = url.split('/')[-1]
self.info("Downloading %s to %s (%s)", url, target, bytes_to_string(size))
start = time.time()
r = self.robust(requests.get)(url, stream=True, verify=self.verify,
timeout=20)
try:
r.raise_for_status()
total = 0
with open(target, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
total += len(chunk)
finally:
r.close()
if total != size:
raise Exception("Download failed: downloaded %s byte(s) out of %s" % (total, size))
elapsed = time.time() - start
if elapsed:
self.info("Download rate %s/s", bytes_to_string(size / elapsed))
return target
def test_bytes_to_string():
assert api.bytes_to_string(1) == '1'
assert api.bytes_to_string(1 << 10) == '1K'
assert api.bytes_to_string(1 << 20) == '1M'
assert api.bytes_to_string(1 << 30) == '1G'
assert api.bytes_to_string(1 << 40) == '1T'
assert api.bytes_to_string(1 << 50) == '1P'