def modelpath4file(filename):
"""
Returns URLs.MODEL path if file exists. Otherwise returns config path"
:param filename:
:return:
"""
local_path = URLs.LOCAL_PATH / 'models' / filename
if local_path.exists():
return local_path
else:
return Config.model_path() / filename
def test_creates_config():
DEFAULT_CONFIG_PATH = 'config_test/test.yml'
try:
config_path = _expand_path(DEFAULT_CONFIG_PATH)
clean_path(config_path)
assert not config_path.exists(), "config path should not exist"
config = Config.get(config_path)
assert config_path.exists(), "Config.get should create config if it doesn't exist"
finally:
clean_path(config_path)
def test_default_config():
this_tests(Config)
Config.DEFAULT_CONFIG_LOCATION = 'config_test'
Config.DEFAULT_CONFIG_PATH = Config.DEFAULT_CONFIG_LOCATION + '/config.yml'
try:
assert Config.get() == {
'data_archive_path': str(_expand_path('~/.fastai/data')),
'data_path': str(_expand_path('~/.fastai/data')),
'model_path': str(_expand_path('~/.fastai/models'))
}
finally:
clean_test_config(Config.DEFAULT_CONFIG_LOCATION)
def test_default_config():
this_tests(Config)
Config.DEFAULT_CONFIG_LOCATION = "config_test"
Config.DEFAULT_CONFIG_PATH = Config.DEFAULT_CONFIG_LOCATION + "/config.yml"
try:
assert Config.get() == {
"data_archive_path": str(_expand_path("~/.fastai/data")),
"data_path": str(_expand_path("~/.fastai/data")),
"model_path": str(_expand_path("~/.fastai/models")),
}
finally:
clean_test_config(Config.DEFAULT_CONFIG_LOCATION)
def test_default_config():
this_tests(Config)
Config.DEFAULT_CONFIG_LOCATION = 'config_test'
Config.DEFAULT_CONFIG_PATH = Config.DEFAULT_CONFIG_LOCATION + '/config.yml'
try:
assert Config.get() == {
'data_archive_path': str(_expand_path('~/.fastai/data')),
'data_path': str(_expand_path('~/.fastai/data')),
'model_path': str(_expand_path('~/.fastai/models'))
}
finally:
clean_test_config(Config.DEFAULT_CONFIG_LOCATION)
def test_creates_config():
this_tests(Config)
DEFAULT_CONFIG_PATH = 'config_test/test.yml'
try:
config_path = _expand_path(DEFAULT_CONFIG_PATH)
clean_test_config(config_path)
assert not config_path.exists(), "config path should not exist"
config = Config.get(config_path)
assert config_path.exists(), "Config.get should create config if it doesn't exist"
finally:
clean_test_config(config_path)
assert not config_path.exists(), "config path should not exist"
def download_url(url: str,
dest: str,
overwrite: bool = False,
pbar: ProgressBar = None,
show_progress=True,
chunk_size=1024 * 1024,
timeout=4,
retries=5) -> None:
"Download `url` to `dest` unless it exists and not `overwrite`."
if os.path.exists(dest) and not overwrite: return
s = requests.Session()
s.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
# additional line to identify as a firefox browser, see #2438
s.headers.update({
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:71.0) Gecko/20100101 Firefox/71.0'
})
u = s.get(url, stream=True, timeout=timeout)
try:
file_size = int(u.headers["Content-Length"])
except:
show_progress = False
with open(dest, 'wb') as f:
nbytes = 0
if show_progress:
pbar = progress_bar(range(file_size),
auto_update=False,
leave=False,
parent=pbar)
try:
for chunk in u.iter_content(chunk_size=chunk_size):
nbytes += len(chunk)
if show_progress: pbar.update(nbytes)
f.write(chunk)
except requests.exceptions.ConnectionError as e:
fname = url.split('/')[-1]
from fastai.datasets import Config
data_dir = Config().data_path()
timeout_txt = (
f'\n Download of {url} has failed after {retries} retries\n'
f' Fix the download manually:\n'
f'$ mkdir -p {data_dir}\n'
f'$ cd {data_dir}\n'
f'$ wget -c {url}\n'
f'$ tar -zxvf {fname}\n\n'
f'And re-run your code once the download is successful\n')
print(timeout_txt)
import sys
sys.exit(1)
def datapath4file(filename, folder):
"Return data path to `filename`, checking locally first then in the config file."
local_path = URLs.LOCAL_PATH / folder
if local_path.exists(): return local_path / filename
else: return Config.data_path() / folder / filename