def download_url(url:str, dest:str, overwrite:bool=False, pbar:ProgressBar=None,
show_progress=True, chunk_size=1024*1024, timeout=4, retries=5)->None:
"Download `url` to `dest` unless it exists and not `overwrite`."
if os.path.exists(dest) and not overwrite: return
s = requests.Session()
s.mount('http://',requests.adapters.HTTPAdapter(max_retries=retries))
u = s.get(url, stream=True, timeout=timeout)
try: file_size = int(u.headers["Content-Length"])
except: show_progress = False
with open(dest, 'wb') as f:
nbytes = 0
if show_progress: pbar = progress_bar(range(file_size), auto_update=False, leave=False, parent=pbar)
try:
for chunk in u.iter_content(chunk_size=chunk_size):
nbytes += len(chunk)
if show_progress: pbar.update(nbytes)
f.write(chunk)
except requests.exceptions.ConnectionError as e:
fname = url.split('/')[-1]
from fastai.datasets import Config
data_dir = Config().data_path()
timeout_txt =(f'\n Download of {url} has failed after {retries} retries\n'
f' Fix the download manually:\n'
f'$ mkdir -p {data_dir}\n'
f'$ cd {data_dir}\n'
f'$ wget -c {url}\n'
f'$ tar -zxvf {fname}\n\n'
f'And re-run your code once the download is successful\n')
print(timeout_txt)
import sys;sys.exit(1)
def download_url(
url: str,
dest: str,
overwrite: bool = False,
pbar: ProgressBar = None,
show_progress=True,
chunk_size=1024 * 1024,
timeout=4,
retries=5,
) -> None:
"Download `url` to `dest` unless it exists and not `overwrite`."
if os.path.exists(dest) and not overwrite:
return
s = requests.Session()
s.mount("http://", requests.adapters.HTTPAdapter(max_retries=retries))
# additional line to identify as a firefox browser, see #2438
s.headers.update(
{
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:71.0) Gecko/20100101 Firefox/71.0"
}
)
u = s.get(url, stream=True, timeout=timeout)
try:
file_size = int(u.headers["Content-Length"])
except:
show_progress = False
with open(dest, "wb") as f:
nbytes = 0
if show_progress:
pbar = progress_bar(range(file_size), leave=False, parent=pbar)
try:
if show_progress:
pbar.update(0)
for chunk in u.iter_content(chunk_size=chunk_size):
nbytes += len(chunk)
if show_progress:
pbar.update(nbytes)
f.write(chunk)
except requests.exceptions.ConnectionError as e:
fname = url.split("/")[-1]
from fastai.datasets import Config
data_dir = Config().data_path()
timeout_txt = (
f"\n Download of {url} has failed after {retries} retries\n"
f" Fix the download manually:\n"
f"$ mkdir -p {data_dir}\n"
f"$ cd {data_dir}\n"
f"$ wget -c {url}\n"
f"$ tar -zxvf {fname}\n\n"
f"And re-run your code once the download is successful\n"
)
print(timeout_txt)
import sys
sys.exit(1)