def modelpath4file(filename): """ Returns URLs.MODEL path if file exists. Otherwise returns config path" :param filename: :return: """ local_path = URLs.LOCAL_PATH / 'models' / filename if local_path.exists(): return local_path else: return Config.model_path() / filename
def test_creates_config(): DEFAULT_CONFIG_PATH = 'config_test/test.yml' try: config_path = _expand_path(DEFAULT_CONFIG_PATH) clean_path(config_path) assert not config_path.exists(), "config path should not exist" config = Config.get(config_path) assert config_path.exists(), "Config.get should create config if it doesn't exist" finally: clean_path(config_path)
def test_default_config(): this_tests(Config) Config.DEFAULT_CONFIG_LOCATION = 'config_test' Config.DEFAULT_CONFIG_PATH = Config.DEFAULT_CONFIG_LOCATION + '/config.yml' try: assert Config.get() == { 'data_archive_path': str(_expand_path('~/.fastai/data')), 'data_path': str(_expand_path('~/.fastai/data')), 'model_path': str(_expand_path('~/.fastai/models')) } finally: clean_test_config(Config.DEFAULT_CONFIG_LOCATION)
def test_default_config(): this_tests(Config) Config.DEFAULT_CONFIG_LOCATION = "config_test" Config.DEFAULT_CONFIG_PATH = Config.DEFAULT_CONFIG_LOCATION + "/config.yml" try: assert Config.get() == { "data_archive_path": str(_expand_path("~/.fastai/data")), "data_path": str(_expand_path("~/.fastai/data")), "model_path": str(_expand_path("~/.fastai/models")), } finally: clean_test_config(Config.DEFAULT_CONFIG_LOCATION)
def test_default_config(): this_tests(Config) Config.DEFAULT_CONFIG_LOCATION = 'config_test' Config.DEFAULT_CONFIG_PATH = Config.DEFAULT_CONFIG_LOCATION + '/config.yml' try: assert Config.get() == { 'data_archive_path': str(_expand_path('~/.fastai/data')), 'data_path': str(_expand_path('~/.fastai/data')), 'model_path': str(_expand_path('~/.fastai/models')) } finally: clean_test_config(Config.DEFAULT_CONFIG_LOCATION)
def test_creates_config(): this_tests(Config) DEFAULT_CONFIG_PATH = 'config_test/test.yml' try: config_path = _expand_path(DEFAULT_CONFIG_PATH) clean_test_config(config_path) assert not config_path.exists(), "config path should not exist" config = Config.get(config_path) assert config_path.exists(), "Config.get should create config if it doesn't exist" finally: clean_test_config(config_path) assert not config_path.exists(), "config path should not exist"
def download_url(url: str, dest: str, overwrite: bool = False, pbar: ProgressBar = None, show_progress=True, chunk_size=1024 * 1024, timeout=4, retries=5) -> None: "Download `url` to `dest` unless it exists and not `overwrite`." if os.path.exists(dest) and not overwrite: return s = requests.Session() s.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries)) # additional line to identify as a firefox browser, see #2438 s.headers.update({ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:71.0) Gecko/20100101 Firefox/71.0' }) u = s.get(url, stream=True, timeout=timeout) try: file_size = int(u.headers["Content-Length"]) except: show_progress = False with open(dest, 'wb') as f: nbytes = 0 if show_progress: pbar = progress_bar(range(file_size), auto_update=False, leave=False, parent=pbar) try: for chunk in u.iter_content(chunk_size=chunk_size): nbytes += len(chunk) if show_progress: pbar.update(nbytes) f.write(chunk) except requests.exceptions.ConnectionError as e: fname = url.split('/')[-1] from fastai.datasets import Config data_dir = Config().data_path() timeout_txt = ( f'\n Download of {url} has failed after {retries} retries\n' f' Fix the download manually:\n' f'$ mkdir -p {data_dir}\n' f'$ cd {data_dir}\n' f'$ wget -c {url}\n' f'$ tar -zxvf {fname}\n\n' f'And re-run your code once the download is successful\n') print(timeout_txt) import sys sys.exit(1)
def datapath4file(filename, folder): "Return data path to `filename`, checking locally first then in the config file." local_path = URLs.LOCAL_PATH / folder if local_path.exists(): return local_path / filename else: return Config.data_path() / folder / filename