def download_sample_data(overwrite=False):
"""
Download all sample data at once. This will overwrite any existing files.
Parameters
----------
overwrite: `bool`
Overwrite existing sample data.
"""
# Workaround for tox only. This is not supported as a user option
sampledata_dir = os.environ.get("SUNPY_SAMPLEDIR", False)
if sampledata_dir:
sampledata_dir = Path(sampledata_dir).expanduser().resolve()
_is_writable_dir(sampledata_dir)
else:
# Creating the directory for sample files to be downloaded
sampledata_dir = Path(get_and_create_sample_dir())
already_downloaded = []
to_download = []
for url_file_name in _SAMPLE_FILES.keys():
fname = sampledata_dir/url_file_name
# We want to avoid calling download if we already have all the files.
if fname.exists() and not overwrite:
already_downloaded.append(fname)
else:
# URL and Filename pairs
to_download.append((url_file_name, fname))
if to_download:
results = _download_sample_data(_BASE_URLS[0], to_download, overwrite=overwrite)
else:
return already_downloaded
# Something went wrong.
if results.errors:
results = _retry_sample_data(results)
return results + already_downloaded
def download_sample_data(overwrite=False):
"""
Download all sample data at once. This will overwrite any existing files.
Parameters
----------
overwrite: `bool`
Overwrite existing sample data.
"""
# Workaround for tox only. This is not supported as a user option
sampledata_dir = os.environ.get("SUNPY_SAMPLEDIR", False)
if sampledata_dir:
sampledata_dir = Path(sampledata_dir).expanduser().resolve()
_is_writable_dir(sampledata_dir)
else:
# Creating the directory for sample files to be downloaded
sampledata_dir = Path(get_and_create_sample_dir())
dl = parfive.Downloader(overwrite=overwrite)
first_url = _base_urls[0]
already_downloaded = []
for file_name in _sample_files.keys():
url = urljoin(first_url, file_name)
fname = sampledata_dir / file_name
# We have to avoid calling download if we already have all the files.
if fname.exists() and not overwrite:
already_downloaded.append(fname)
else:
dl.enqueue_file(url, filename=sampledata_dir / file_name)
if dl.queued_downloads:
results = dl.download()
else:
return already_downloaded
if not results.errors:
return results
for retry_url in _base_urls[1:]:
for i, err in enumerate(results.errors):
file_name = Path(err.url).name
# Overwrite the parfive error to change the url to a mirror
new_url = urljoin(retry_url, file_name)
results._errors[i] = _error(err.filepath_partial, new_url,
err.exception)
results = dl.retry(results)
if not results.errors:
return results
for err in results.errors:
file_name = Path(err.url).name
warnings.warn(f"File {file_name} not found.", SunpyUserWarning)
return results
def test_is_writable_dir(tmpdir, tmp_path):
assert _is_writable_dir(tmpdir)
tmp_file = tmpdir.join("hello.txt")
# Have to write to the file otherwise its seen as a directory(?!)
tmp_file.write("content")
# Checks directory with a file
assert _is_writable_dir(tmpdir)
# Checks a filepath instead of directory
assert not _is_writable_dir(tmp_file)
def test_is_writable_dir(tmpdir, tmp_path):
assert _is_writable_dir(tmpdir)
def download_sample_data(overwrite=False):
"""
Download all sample data at once. This will overwrite any existing files.
Parameters
----------
overwrite: `bool`
Overwrite existing sample data.
"""
# Workaround for tox only. This is not supported as a user option
sampledata_dir = os.environ.get("SUNPY_SAMPLEDIR", False)
if sampledata_dir:
sampledata_dir = Path(sampledata_dir).expanduser().resolve()
_is_writable_dir(sampledata_dir)
else:
# Creating the directory for sample files to be downloaded
sampledata_dir = Path(get_and_create_sample_dir())
dl = Downloader(overwrite=overwrite)
first_url = _base_urls[0]
already_downloaded = []
for file_name in _sample_files.keys():
url = urljoin(first_url, file_name)
fname = sampledata_dir / file_name
# We have to avoid calling download if we already have all the files.
if fname.exists() and not overwrite:
already_downloaded.append(fname)
else:
dl.enqueue_file(url, filename=fname)
if dl.queued_downloads:
results = dl.download()
else:
return already_downloaded
if not results.errors:
return results
else:
log.info(
'Failed to download one or more sample data files, retrying with a mirror.'
)
for retry_url in _base_urls[1:]:
for i, err in enumerate(results.errors):
file_name = err.filepath_partial().name
log.debug(
f"Failed to download {_sample_files[file_name]} from {err.url}: {err.exception}"
)
# Overwrite the parfive error to change the url to a mirror
new_url = urljoin(retry_url, file_name)
results._errors[i] = _error(err.filepath_partial, new_url,
err.exception)
results = dl.retry(results)
if not results.errors:
return results
for err in results.errors:
file_name = err.filepath_partial().name
log.debug(
f"Failed to download {_sample_files[file_name]} from {err.url}: {err.exception}"
)
log.error(
f"Failed to download {_sample_files[file_name]} from all mirrors, the file will not be available."
)
return results
def test_is_writable_dir(tmpdir, tmp_path):
writeable_dir = tmpdir.mkdir("can_you_right_to_me")
just_a_path = tmp_path / "sub"
assert _is_writable_dir(writeable_dir)
assert not _is_writable_dir(just_a_path)
def test_is_writable_dir(tmpdir, tmp_path):
assert _is_writable_dir(tmpdir)
def test_is_writable_dir(tmpdir, tmp_path):
writeable_dir = tmpdir.mkdir("can_you_right_to_me")
just_a_path = tmp_path / "sub"
assert _is_writable_dir(writeable_dir)
assert not _is_writable_dir(just_a_path)