def save_url(url, name, reporthook=simple_hook): bs = 1024 * 8 size = -1 read = 0 blocknum = 0 open_mode = 'wb' req = Request(url, headers=fake_headers) response = urlopen(req, None) if "content-length" in response.headers: size = int(response.headers["Content-Length"]) if os.path.exists(name): filesize = os.path.getsize(name) if filesize == size: print('Skipped: file already downloaded') return elif -1 != size: req.add_header('Range', 'bytes=%d-' % filesize) blocknum = int(filesize / bs) response = urlopen(req, None) open_mode = 'ab' reporthook(blocknum, bs, size) with open(name, open_mode) as tfp: while True: block = response.read(bs) if not block: break read += len(block) tfp.write(block) blocknum += 1 reporthook(blocknum, bs, size)
def save_url(url, name, reporthook = simple_hook): bs = 1024*8 size = -1 read = 0 blocknum = 0 open_mode = 'wb' req = Request(url, headers = fake_headers) response = urlopen(req, None) if "content-length" in response.headers: size = int(response.headers["Content-Length"]) if os.path.exists(name): filesize = os.path.getsize(name) if filesize == size: print('Skipped: file already downloaded') return elif -1 != size: req.add_header('Range', 'bytes=%d-' % filesize) blocknum = int(filesize / bs) response = urlopen(req, None) open_mode = 'ab' reporthook(blocknum, bs, size) with open(name, open_mode) as tfp: while True: block = response.read(bs) if not block: break read += len(block) tfp.write(block) blocknum += 1 reporthook(blocknum, bs, size)
def save_url(url, name, ext, status, part=None, reporthook=simple_hook): if part is None: print('Download: ' + name) name = name + '.' + ext part = 0 else: print('\nDownload: %s part %d' % (name, part)) name = '%s_%d.%s' % (name, part, ext) bs = 1024 * 8 size = -1 read = 0 blocknum = 0 open_mode = 'wb' response = None req = Request(url, headers=fake_headers) if os.path.exists(name): filesize = os.path.getsize(name) if filesize: req.add_header('Range', 'bytes=%d-' % (filesize - 1)) # get +1, avoid 416 response = urlopen(req, None) assert response.status == 206, 'HTTP status %d' % response.status size = int(response.headers['Content-Range'].split('/')[-1]) if filesize == size: print('Skipped: file already downloaded') status[part] = 1 return if filesize < size: if filesize: blocknum = int(filesize / bs) open_mode = 'ab' response.read(1) # read -1 if response is None: response = urlopen(req, None) if size < 0: size = int(response.headers.get('Content-Length', -1)) reporthook(blocknum, bs, size) with open(name, open_mode) as tfp: while True: block = response.read(bs) if not block: break read += len(block) tfp.write(block) blocknum += 1 reporthook(blocknum, bs, size) if os.path.exists(name): filesize = os.path.getsize(name) if filesize == size: status[part] = 1
def save_url(url, name, ext, status, part=None, reporthook=simple_hook): if part is None: print("Download: " + name) name = name + '.' + ext part = 0 else: print("Download: " + name + " part %d" % part) name = name + '_%d_.' % part + ext bs = 1024 * 8 size = -1 read = 0 blocknum = 0 open_mode = 'wb' req = Request(url, headers=fake_headers) if os.path.exists(name): filesize = os.path.getsize(name) req.add_header('Range', 'bytes=%d-' % filesize) response = urlopen(req, None) if response.status == 206: size = int(response.headers['Content-Range'].split('/')[-1]) if filesize == size: print('Skipped: file already downloaded') status[part] = 1 return if filesize < size: if filesize: blocknum = int(filesize / bs) open_mode = 'ab' else: response = urlopen(req, None) if size < 0: size = int(response.headers.get('Content-Length', -1)) reporthook(blocknum, bs, size) with open(name, open_mode) as tfp: while True: block = response.read(bs) if not block: break read += len(block) tfp.write(block) blocknum += 1 reporthook(blocknum, bs, size) if os.path.exists(name): filesize = os.path.getsize(name) if filesize == size: status[part] = 1
def _save_url(url, name, ext, status, part=None, reporthook=multi_hook): def print(*args, **kwargs): reporthook(['print', args, kwargs]) def read_response(bs): if size > 0: # a independent timeout for read response rd, _, ed = select.select([fd], [], [fd], timeout) if ed: raise socket.error(ed) if not rd: raise socket.timeout('The read operation timed out') return response.read(bs) if part is None: name = name + '.' + ext part = 0 else: name = '%s_%d.%s' % (name, part, ext) bs = 8192 size = -1 filesize = 0 downloaded = 0 open_mode = 'wb' response = None timeout = max(socket.getdefaulttimeout() or 0, 60) req = Request(url, headers=fake_headers) try: reporthook(['part'], part=part) if os.path.exists(name): filesize = os.path.getsize(name) if filesize: req.add_header('Range', 'bytes=%d-' % (filesize - 1)) # get +1, avoid 416 response = urlopen(req, None) set_rcvbuf(response) if response.status == 206: size = int( response.headers['Content-Range'].split('/')[-1]) needless_size = 1 elif response.status == 200: size = int(response.headers.get('Content-Length', -1)) needless_size = filesize if filesize == size: print('Skipped: file part %d has already been downloaded' % part) status[part] = 1 return True if filesize < size: percent = int(filesize * 100 / size) open_mode = 'ab' print('Restored: file part %d is incomplete at %d%%' % (part, percent)) reporthook(['part'], filesize, size, part) fd = response.fileno() while needless_size > 0: if needless_size > bs: block = read_response(bs) else: block = read_response(needless_size) if not block: return needless_size -= len(block) if response is None: response = urlopen(req, None) set_rcvbuf(response) fd = response.fileno() if size < 0: size = int(response.headers.get('Content-Length', -1)) with open(name, open_mode) as tfp: while size < 0 or filesize < size: block = read_response(bs) if not block: break n = tfp.write(block) downloaded += n filesize += n reporthook(['part'], filesize, size, part) if os.path.exists(name): filesize = os.path.getsize(name) if filesize and (size < 0 or filesize == size): status[part] = 1 return True finally: time.sleep(1) reporthook(['part end', status, downloaded], filesize, size, part)