def callback(filename, data, size, chunk, chunk_size):
have = len(data) + len(chunk)
if size == 0:
size = '???'
percent = '???'
else:
percent = int(100 * have // size)
size = misc.size(size)
have = misc.size(have)
w.status('%s/%s (%s%%)' % (have, size, percent))
return True
def callback(filename, data, size, chunk, chunk_size):
have = len(data) + len(chunk)
if size == 0:
size = '???'
percent = '???'
else:
percent = int(100 * have // size)
size = misc.size(size)
have = misc.size(have)
w.status('%s/%s (%s%%)' % (have, size, percent))
return True
def recvall(self, timeout=Timeout.forever):
"""recvall() -> str
Receives data until EOF is reached.
"""
with self.waitfor('Receiving all data') as h:
l = len(self.buffer)
with self.local(timeout):
try:
while True:
l = misc.size(len(self.buffer))
h.status(l)
if not self._fillbuffer():
break
except EOFError:
pass
h.success("Done (%s)" % l)
self.close()
return self.buffer.get()
def wget(url, save=None, timeout=5, **kwargs):
r"""wget(url, save=None, timeout=5) -> str
Downloads a file via HTTP/HTTPS.
Arguments:
url (str): URL to download
save (str or bool): Name to save as. Any truthy value
will auto-generate a name based on the URL.
timeout (int): Timeout, in seconds
Example:
>>> url = 'https://httpbin.org/robots.txt'
>>> result = wget(url, timeout=60)
>>> result
b'User-agent: *\nDisallow: /deny\n'
>>> filename = tempfile.mktemp()
>>> result2 = wget(url, filename, timeout=60)
>>> result == open(filename, 'rb').read()
True
"""
import requests
with log.progress("Downloading '%s'" % url, rate=0.1) as w:
w.status("Making request...")
response = requests.get(url, stream=True, timeout=timeout, **kwargs)
if not response.ok:
w.failure("Got code %s" % response.status_code)
return
total_size = int(response.headers.get('content-length', 0))
w.status('0 / %s' % size(total_size))
# Find out the next largest size we can represent as
chunk_size = 1
while chunk_size < (total_size / 10):
chunk_size *= 1000
# Count chunks as they're received
buf = Buffer()
# Loop until we have all of the data
for chunk in response.iter_content(chunk_size=2**10):
buf.add(chunk)
if total_size:
w.status('%s / %s' % (size(buf.size), size(total_size)))
else:
w.status('%s' % size(buf.size))
total_data = buf.get()
# Save to the target file if provided
if save:
if not isinstance(save, (bytes, six.text_type)):
save = os.path.basename(url)
save = save or tempfile.NamedTemporaryFile(dir='.',
delete=False).name
with open(save, 'wb+') as f:
f.write(total_data)
w.success('Saved %r (%s)' % (f.name, size(total_data)))
else:
w.success('%s' % size(total_data))
return total_data
def wget(url, save=None, timeout=5, **kwargs):
"""wget(url, save=None, timeout=5) -> str
Downloads a file via HTTP/HTTPS.
Arguments:
url (str): URL to download
save (str or bool): Name to save as. Any truthy value
will auto-generate a name based on the URL.
timeout (int): Timeout, in seconds
Example:
>>> url = 'https://httpbin.org/robots.txt'
>>> result = wget(url, timeout=60)
>>> result
'User-agent: *\\nDisallow: /deny\\n'
>>> result2 = wget(url, True, timeout=60)
>>> result == file('robots.txt').read()
True
"""
import requests
with log.progress("Downloading '%s'" % url, rate=0.1) as w:
w.status("Making request...")
response = requests.get(url, stream=True, timeout=timeout, **kwargs)
if not response.ok:
w.failure("Got code %s" % response.status_code)
return
total_size = int(response.headers.get('content-length',0))
w.status('0 / %s' % size(total_size))
# Find out the next largest size we can represent as
chunk_size = 1
while chunk_size < (total_size/10):
chunk_size *= 1000
# Count chunks as they're received
buf = Buffer()
# Loop until we have all of the data
for chunk in response.iter_content(chunk_size = 2**10):
buf.add(chunk)
if total_size:
w.status('%s / %s' % (size(buf.size), size(total_size)))
else:
w.status('%s' % size(buf.size))
total_data = buf.get()
# Save to the target file if provided
if save:
if not isinstance(save, (str, unicode)):
save = os.path.basename(url)
save = save or tempfile.NamedTemporaryFile(dir='.', delete=False).name
with file(save,'wb+') as f:
f.write(total_data)
w.success('Saved %r (%s)' % (f.name, size(total_data)))
else:
w.success('%s' % size(total_data))
return total_data
