def save_url(url, name, reporthook=simple_hook):
bs = 1024 * 8
size = -1
read = 0
blocknum = 0
open_mode = 'wb'
req = Request(url, headers=fake_headers)
response = urlopen(req, None)
if "content-length" in response.headers:
size = int(response.headers["Content-Length"])
if os.path.exists(name):
filesize = os.path.getsize(name)
if filesize == size:
print('Skipped: file already downloaded')
return
elif -1 != size:
req.add_header('Range', 'bytes=%d-' % filesize)
blocknum = int(filesize / bs)
response = urlopen(req, None)
open_mode = 'ab'
reporthook(blocknum, bs, size)
with open(name, open_mode) as tfp:
while True:
block = response.read(bs)
if not block:
break
read += len(block)
tfp.write(block)
blocknum += 1
reporthook(blocknum, bs, size)
def save_url(url, name, reporthook = simple_hook):
bs = 1024*8
size = -1
read = 0
blocknum = 0
open_mode = 'wb'
req = Request(url, headers = fake_headers)
response = urlopen(req, None)
if "content-length" in response.headers:
size = int(response.headers["Content-Length"])
if os.path.exists(name):
filesize = os.path.getsize(name)
if filesize == size:
print('Skipped: file already downloaded')
return
elif -1 != size:
req.add_header('Range', 'bytes=%d-' % filesize)
blocknum = int(filesize / bs)
response = urlopen(req, None)
open_mode = 'ab'
reporthook(blocknum, bs, size)
with open(name, open_mode) as tfp:
while True:
block = response.read(bs)
if not block:
break
read += len(block)
tfp.write(block)
blocknum += 1
reporthook(blocknum, bs, size)
def save_url(url, name, ext, status, part=None, reporthook=simple_hook):
if part is None:
print('Download: ' + name)
name = name + '.' + ext
part = 0
else:
print('\nDownload: %s part %d' % (name, part))
name = '%s_%d.%s' % (name, part, ext)
bs = 1024 * 8
size = -1
read = 0
blocknum = 0
open_mode = 'wb'
response = None
req = Request(url, headers=fake_headers)
if os.path.exists(name):
filesize = os.path.getsize(name)
if filesize:
req.add_header('Range',
'bytes=%d-' % (filesize - 1)) # get +1, avoid 416
response = urlopen(req, None)
assert response.status == 206, 'HTTP status %d' % response.status
size = int(response.headers['Content-Range'].split('/')[-1])
if filesize == size:
print('Skipped: file already downloaded')
status[part] = 1
return
if filesize < size:
if filesize:
blocknum = int(filesize / bs)
open_mode = 'ab'
response.read(1) # read -1
if response is None:
response = urlopen(req, None)
if size < 0:
size = int(response.headers.get('Content-Length', -1))
reporthook(blocknum, bs, size)
with open(name, open_mode) as tfp:
while True:
block = response.read(bs)
if not block:
break
read += len(block)
tfp.write(block)
blocknum += 1
reporthook(blocknum, bs, size)
if os.path.exists(name):
filesize = os.path.getsize(name)
if filesize == size:
status[part] = 1
def save_url(url, name, ext, status, part=None, reporthook=simple_hook):
if part is None:
print("Download: " + name)
name = name + '.' + ext
part = 0
else:
print("Download: " + name + " part %d" % part)
name = name + '_%d_.' % part + ext
bs = 1024 * 8
size = -1
read = 0
blocknum = 0
open_mode = 'wb'
req = Request(url, headers=fake_headers)
if os.path.exists(name):
filesize = os.path.getsize(name)
req.add_header('Range', 'bytes=%d-' % filesize)
response = urlopen(req, None)
if response.status == 206:
size = int(response.headers['Content-Range'].split('/')[-1])
if filesize == size:
print('Skipped: file already downloaded')
status[part] = 1
return
if filesize < size:
if filesize:
blocknum = int(filesize / bs)
open_mode = 'ab'
else:
response = urlopen(req, None)
if size < 0:
size = int(response.headers.get('Content-Length', -1))
reporthook(blocknum, bs, size)
with open(name, open_mode) as tfp:
while True:
block = response.read(bs)
if not block:
break
read += len(block)
tfp.write(block)
blocknum += 1
reporthook(blocknum, bs, size)
if os.path.exists(name):
filesize = os.path.getsize(name)
if filesize == size:
status[part] = 1
def _save_url(url, name, ext, status, part=None, reporthook=multi_hook):
def print(*args, **kwargs):
reporthook(['print', args, kwargs])
def read_response(bs):
if size > 0:
# a independent timeout for read response
rd, _, ed = select.select([fd], [], [fd], timeout)
if ed:
raise socket.error(ed)
if not rd:
raise socket.timeout('The read operation timed out')
return response.read(bs)
if part is None:
name = name + '.' + ext
part = 0
else:
name = '%s_%d.%s' % (name, part, ext)
bs = 8192
size = -1
filesize = 0
downloaded = 0
open_mode = 'wb'
response = None
timeout = max(socket.getdefaulttimeout() or 0, 60)
req = Request(url, headers=fake_headers)
try:
reporthook(['part'], part=part)
if os.path.exists(name):
filesize = os.path.getsize(name)
if filesize:
req.add_header('Range', 'bytes=%d-' %
(filesize - 1)) # get +1, avoid 416
response = urlopen(req, None)
set_rcvbuf(response)
if response.status == 206:
size = int(
response.headers['Content-Range'].split('/')[-1])
needless_size = 1
elif response.status == 200:
size = int(response.headers.get('Content-Length', -1))
needless_size = filesize
if filesize == size:
print('Skipped: file part %d has already been downloaded' %
part)
status[part] = 1
return True
if filesize < size:
percent = int(filesize * 100 / size)
open_mode = 'ab'
print('Restored: file part %d is incomplete at %d%%' %
(part, percent))
reporthook(['part'], filesize, size, part)
fd = response.fileno()
while needless_size > 0:
if needless_size > bs:
block = read_response(bs)
else:
block = read_response(needless_size)
if not block:
return
needless_size -= len(block)
if response is None:
response = urlopen(req, None)
set_rcvbuf(response)
fd = response.fileno()
if size < 0:
size = int(response.headers.get('Content-Length', -1))
with open(name, open_mode) as tfp:
while size < 0 or filesize < size:
block = read_response(bs)
if not block:
break
n = tfp.write(block)
downloaded += n
filesize += n
reporthook(['part'], filesize, size, part)
if os.path.exists(name):
filesize = os.path.getsize(name)
if filesize and (size < 0 or filesize == size):
status[part] = 1
return True
finally:
time.sleep(1)
reporthook(['part end', status, downloaded], filesize, size, part)
