def download_sample_data(overwrite=False): """ Download all sample data at once. This will overwrite any existing files. Parameters ---------- overwrite: `bool` Overwrite existing sample data. """ # Workaround for tox only. This is not supported as a user option sampledata_dir = os.environ.get("SUNPY_SAMPLEDIR", False) if sampledata_dir: sampledata_dir = Path(sampledata_dir).expanduser().resolve() _is_writable_dir(sampledata_dir) else: # Creating the directory for sample files to be downloaded sampledata_dir = Path(get_and_create_sample_dir()) already_downloaded = [] to_download = [] for url_file_name in _SAMPLE_FILES.keys(): fname = sampledata_dir/url_file_name # We want to avoid calling download if we already have all the files. if fname.exists() and not overwrite: already_downloaded.append(fname) else: # URL and Filename pairs to_download.append((url_file_name, fname)) if to_download: results = _download_sample_data(_BASE_URLS[0], to_download, overwrite=overwrite) else: return already_downloaded # Something went wrong. if results.errors: results = _retry_sample_data(results) return results + already_downloaded
def download_sample_data(overwrite=False): """ Download all sample data at once. This will overwrite any existing files. Parameters ---------- overwrite: `bool` Overwrite existing sample data. """ # Workaround for tox only. This is not supported as a user option sampledata_dir = os.environ.get("SUNPY_SAMPLEDIR", False) if sampledata_dir: sampledata_dir = Path(sampledata_dir).expanduser().resolve() _is_writable_dir(sampledata_dir) else: # Creating the directory for sample files to be downloaded sampledata_dir = Path(get_and_create_sample_dir()) dl = parfive.Downloader(overwrite=overwrite) first_url = _base_urls[0] already_downloaded = [] for file_name in _sample_files.keys(): url = urljoin(first_url, file_name) fname = sampledata_dir / file_name # We have to avoid calling download if we already have all the files. if fname.exists() and not overwrite: already_downloaded.append(fname) else: dl.enqueue_file(url, filename=sampledata_dir / file_name) if dl.queued_downloads: results = dl.download() else: return already_downloaded if not results.errors: return results for retry_url in _base_urls[1:]: for i, err in enumerate(results.errors): file_name = Path(err.url).name # Overwrite the parfive error to change the url to a mirror new_url = urljoin(retry_url, file_name) results._errors[i] = _error(err.filepath_partial, new_url, err.exception) results = dl.retry(results) if not results.errors: return results for err in results.errors: file_name = Path(err.url).name warnings.warn(f"File {file_name} not found.", SunpyUserWarning) return results
def test_is_writable_dir(tmpdir, tmp_path): assert _is_writable_dir(tmpdir) tmp_file = tmpdir.join("hello.txt") # Have to write to the file otherwise its seen as a directory(?!) tmp_file.write("content") # Checks directory with a file assert _is_writable_dir(tmpdir) # Checks a filepath instead of directory assert not _is_writable_dir(tmp_file)
def test_is_writable_dir(tmpdir, tmp_path): assert _is_writable_dir(tmpdir)
def download_sample_data(overwrite=False): """ Download all sample data at once. This will overwrite any existing files. Parameters ---------- overwrite: `bool` Overwrite existing sample data. """ # Workaround for tox only. This is not supported as a user option sampledata_dir = os.environ.get("SUNPY_SAMPLEDIR", False) if sampledata_dir: sampledata_dir = Path(sampledata_dir).expanduser().resolve() _is_writable_dir(sampledata_dir) else: # Creating the directory for sample files to be downloaded sampledata_dir = Path(get_and_create_sample_dir()) dl = Downloader(overwrite=overwrite) first_url = _base_urls[0] already_downloaded = [] for file_name in _sample_files.keys(): url = urljoin(first_url, file_name) fname = sampledata_dir / file_name # We have to avoid calling download if we already have all the files. if fname.exists() and not overwrite: already_downloaded.append(fname) else: dl.enqueue_file(url, filename=fname) if dl.queued_downloads: results = dl.download() else: return already_downloaded if not results.errors: return results else: log.info( 'Failed to download one or more sample data files, retrying with a mirror.' ) for retry_url in _base_urls[1:]: for i, err in enumerate(results.errors): file_name = err.filepath_partial().name log.debug( f"Failed to download {_sample_files[file_name]} from {err.url}: {err.exception}" ) # Overwrite the parfive error to change the url to a mirror new_url = urljoin(retry_url, file_name) results._errors[i] = _error(err.filepath_partial, new_url, err.exception) results = dl.retry(results) if not results.errors: return results for err in results.errors: file_name = err.filepath_partial().name log.debug( f"Failed to download {_sample_files[file_name]} from {err.url}: {err.exception}" ) log.error( f"Failed to download {_sample_files[file_name]} from all mirrors, the file will not be available." ) return results
def test_is_writable_dir(tmpdir, tmp_path): writeable_dir = tmpdir.mkdir("can_you_right_to_me") just_a_path = tmp_path / "sub" assert _is_writable_dir(writeable_dir) assert not _is_writable_dir(just_a_path)
def test_is_writable_dir(tmpdir, tmp_path): assert _is_writable_dir(tmpdir)
def test_is_writable_dir(tmpdir, tmp_path): writeable_dir = tmpdir.mkdir("can_you_right_to_me") just_a_path = tmp_path / "sub" assert _is_writable_dir(writeable_dir) assert not _is_writable_dir(just_a_path)