def test_name_resolve_cache(tmpdir):
from astropy.utils.data import _get_download_cache_locs, get_cached_urls
import shelve
target_name = "castor"
temp_cache_dir = str(tmpdir.mkdir('cache'))
with paths.set_temp_cache(temp_cache_dir, delete=True):
download_dir, urlmapfn = _get_download_cache_locs()
with shelve.open(urlmapfn) as url2hash:
assert len(url2hash) == 0
icrs1 = get_icrs_coordinates(target_name, cache=True)
# This is a weak test: we just check to see that a url is added to the
# cache!
with shelve.open(urlmapfn) as url2hash:
assert len(url2hash) == 1
url = get_cached_urls()[0]
assert 'http://cdsweb.u-strasbg.fr/cgi-bin/nph-sesame/' in url
# Try reloading coordinates, now should just reload cached data:
with no_internet():
icrs2 = get_icrs_coordinates(target_name, cache=True)
with shelve.open(urlmapfn) as url2hash:
assert len(url2hash) == 1
assert u.allclose(icrs1.ra, icrs2.ra)
assert u.allclose(icrs1.dec, icrs2.dec)
def test_download_cache():
from astropy.utils.data import download_file, clear_download_cache
download_dir = _get_download_cache_locs()[0]
# Download the test URL and make sure it exists, then clear just that
# URL and make sure it got deleted.
fnout = download_file(TESTURL, cache=True)
assert os.path.isdir(download_dir)
assert os.path.isfile(fnout)
clear_download_cache(TESTURL)
assert not os.path.exists(fnout)
# Test issues raised in #4427 with clear_download_cache() without a URL,
# followed by subsequent download.
fnout = download_file(TESTURL, cache=True)
assert os.path.isfile(fnout)
clear_download_cache()
assert not os.path.exists(fnout)
assert not os.path.exists(download_dir)
fnout = download_file(TESTURL, cache=True)
assert os.path.isfile(fnout)
# Clearing download cache succeeds even if the URL does not exist.
clear_download_cache('http://this_was_never_downloaded_before.com')
# Make sure lockdir was released
lockdir = os.path.join(download_dir, 'lock')
assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
def test_download_cache():
from astropy.utils.data import download_file, clear_download_cache
download_dir = _get_download_cache_locs()[0]
# Download the test URL and make sure it exists, then clear just that
# URL and make sure it got deleted.
fnout = download_file(TESTURL, cache=True)
assert os.path.isdir(download_dir)
assert os.path.isfile(fnout)
clear_download_cache(TESTURL)
assert not os.path.exists(fnout)
# Test issues raised in #4427 with clear_download_cache() without a URL,
# followed by subsequent download.
fnout = download_file(TESTURL, cache=True)
assert os.path.isfile(fnout)
clear_download_cache()
assert not os.path.exists(fnout)
assert not os.path.exists(download_dir)
fnout = download_file(TESTURL, cache=True)
assert os.path.isfile(fnout)
# Clearing download cache succeeds even if the URL does not exist.
clear_download_cache('http://this_was_never_downloaded_before.com')
# Make sure lockdir was released
lockdir = os.path.join(download_dir, 'lock')
assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
def test_find_by_hash():
from astropy.utils.data import clear_download_cache
with get_readable_fileobj(TESTURL, encoding="binary", cache=True) as page:
hash = hashlib.md5(page.read())
hashstr = 'hash/' + hash.hexdigest()
fnout = get_pkg_data_filename(hashstr)
assert os.path.isfile(fnout)
clear_download_cache(hashstr[5:])
assert not os.path.isfile(fnout)
lockdir = os.path.join(_get_download_cache_locs()[0], 'lock')
assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
def test_find_by_hash():
from astropy.utils.data import clear_download_cache
with get_readable_fileobj(TESTURL, encoding="binary", cache=True) as page:
hash = hashlib.md5(page.read())
hashstr = 'hash/' + hash.hexdigest()
fnout = get_pkg_data_filename(hashstr)
assert os.path.isfile(fnout)
clear_download_cache(hashstr[5:])
assert not os.path.isfile(fnout)
lockdir = os.path.join(_get_download_cache_locs()[0], 'lock')
assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
def test_download_mirror_cache():
import pathlib
import shelve
from astropy.utils.data import _find_pkg_data_path, download_file, get_cached_urls
main_url = pathlib.Path(
_find_pkg_data_path(os.path.join('data', 'dataurl'))).as_uri() + '/'
mirror_url = pathlib.Path(
_find_pkg_data_path(os.path.join(
'data', 'dataurl_mirror'))).as_uri() + '/' # noqa
main_file = main_url + 'index.html'
mirror_file = mirror_url + 'index.html'
# Temporarily change data.conf.
# This also test https://github.com/astropy/astropy/pull/8163 because
# urlopen() on a local dir URI also gives URLError.
with conf.set_temp('dataurl', main_url):
with conf.set_temp('dataurl_mirror', mirror_url):
# "Download" files by rerouting URLs to local URIs.
download_file(main_file, cache=True)
download_file(mirror_file, cache=True)
# Now test that download_file looks in mirror's cache before
# download.
# https://github.com/astropy/astropy/issues/6982
dldir, urlmapfn = _get_download_cache_locs()
with shelve.open(urlmapfn) as url2hash:
del url2hash[main_file]
# Comparing hash makes sure they download the same file
# but does not guarantee they were downloaded from the same URL.
assert (download_file(main_file,
cache=True) == download_file(mirror_file,
cache=True))
# This has to be called after the last download to obtain
# an accurate view of cached URLs.
# This is to ensure that main_file was not re-downloaded
# unnecessarily.
# This test also tests for "assert TESTURL in get_cached_urls()".
c_urls = get_cached_urls()
assert (mirror_file in c_urls) and (main_file not in c_urls)
def IERS_A_in_cache():
"""
Check if the IERS Bulletin A table is locally cached.
"""
url_key = iers.IERS_A_URL
# The below code which accesses ``urlmapfn`` is stolen from
# astropy.utils.data.download_file()
try:
dldir, urlmapfn = _get_download_cache_locs()
except (IOError, OSError) as e:
msg = 'Remote data cache could not be accessed due to '
estr = '' if len(e.args) < 1 else (': ' + str(e))
warnings.warn(CacheMissingWarning(msg + e.__class__.__name__ + estr))
return False
with _open_shelve(urlmapfn, True) as url2hash:
# TODO: try to figure out how to test this in the unicode case
if str(url_key) in url2hash:
return True
return False
def save_fits(self, savepath, link_cache='hard'):
"""
Save a FITS file to savepath
Parameters
----------
savepath : str
The full path to a FITS filename, e.g. "file.fits", or
"/path/to/file.fits".
link_cache : 'hard', 'sym', or False
Try to create a hard or symbolic link to the astropy cached file?
If the system is unable to create a hardlink, the file will be
copied to the target location.
"""
from warnings import warn
self.get_fits()
try:
dldir, urlmapfn = aud._get_download_cache_locs()
except (IOError, OSError) as e:
msg = 'Remote data cache could not be accessed due to '
estr = '' if len(e.args) < 1 else (': ' + str(e))
warn(aud.CacheMissingWarning(msg + e.__class__.__name__ + estr))
with _open_shelve(urlmapfn, True) as url2hash:
if str(self._target) in url2hash:
target = url2hash[str(self._target)]
else:
raise IOError("Cached file not found / does not exist.")
if link_cache == 'hard':
try:
os.link(target, savepath)
except (IOError, OSError, AttributeError):
shutil.copy(target, savepath)
elif link_cache == 'sym':
try:
os.symlink(target, savepath)
except AttributeError:
raise OSError('Creating symlinks is not possible on this OS.')
else:
shutil.copy(target, savepath)
def IERS_A_in_cache():
"""
Check if the IERS Bulletin A table is locally cached.
"""
url_key = iers.IERS_A_URL
# The below code which accesses ``urlmapfn`` is stolen from
# astropy.utils.data.download_file()
try:
dldir, urlmapfn = _get_download_cache_locs()
except (IOError, OSError) as e:
msg = 'Remote data cache could not be accessed due to '
estr = '' if len(e.args) < 1 else (': ' + str(e))
warnings.warn(CacheMissingWarning(msg + e.__class__.__name__ + estr))
return False
with _open_shelve(urlmapfn, True) as url2hash:
# TODO: try to figure out how to test this in the unicode case
if str(url_key) in url2hash:
return True
return False
def save_fits(self, savepath, link_cache='hard'):
"""
Save a FITS file to savepath
Parameters
----------
savepath : str
The full path to a FITS filename, e.g. "file.fits", or
"/path/to/file.fits".
link_cache : 'hard', 'sym', or False
Try to create a hard or symbolic link to the astropy cached file?
If the system is unable to create a hardlink, the file will be
copied to the target location.
"""
from warnings import warn
self.get_fits()
try:
dldir, urlmapfn = aud._get_download_cache_locs()
except (IOError, OSError) as e:
msg = 'Remote data cache could not be accessed due to '
estr = '' if len(e.args) < 1 else (': ' + str(e))
warn(aud.CacheMissingWarning(msg + e.__class__.__name__ + estr))
with aud._open_shelve(urlmapfn, True) as url2hash:
if str(self._target) in url2hash:
target = url2hash[str(self._target)]
else:
raise IOError("Cached file not found / does not exist.")
if link_cache == 'hard':
try:
os.link(target, savepath)
except (IOError, OSError, AttributeError) as e:
shutil.copy(target, savepath)
elif link_cache == 'sym':
try:
os.symlink(target, savepath)
except AttributeError:
raise OSError('Creating symlinks is not possible on this OS.')
else:
shutil.copy(target, savepath)
def test_download_mirror_cache():
import pathlib
import shelve
from astropy.utils.data import _find_pkg_data_path, download_file, get_cached_urls
main_url = pathlib.Path(
_find_pkg_data_path(os.path.join('data', 'dataurl'))).as_uri() + '/'
mirror_url = pathlib.Path(
_find_pkg_data_path(os.path.join('data', 'dataurl_mirror'))).as_uri() + '/' # noqa
main_file = main_url + 'index.html'
mirror_file = mirror_url + 'index.html'
# Temporarily change data.conf.
# This also test https://github.com/astropy/astropy/pull/8163 because
# urlopen() on a local dir URI also gives URLError.
with conf.set_temp('dataurl', main_url):
with conf.set_temp('dataurl_mirror', mirror_url):
# "Download" files by rerouting URLs to local URIs.
download_file(main_file, cache=True)
download_file(mirror_file, cache=True)
# Now test that download_file looks in mirror's cache before
# download.
# https://github.com/astropy/astropy/issues/6982
dldir, urlmapfn = _get_download_cache_locs()
with shelve.open(urlmapfn) as url2hash:
del url2hash[main_file]
# Comparing hash makes sure they download the same file
# but does not guarantee they were downloaded from the same URL.
assert (download_file(main_file, cache=True) ==
download_file(mirror_file, cache=True))
# This has to be called after the last download to obtain
# an accurate view of cached URLs.
# This is to ensure that main_file was not re-downloaded
# unnecessarily.
# This test also tests for "assert TESTURL in get_cached_urls()".
c_urls = get_cached_urls()
assert (mirror_file in c_urls) and (main_file not in c_urls)
def test_data_noastropy_fallback(monkeypatch):
"""
Tests to make sure the default behavior when the cache directory can't
be located is correct
"""
from astropy.utils import data
from astropy.config import paths
# needed for testing the *real* lock at the end
lockdir = os.path.join(_get_download_cache_locs()[0], 'lock')
# better yet, set the configuration to make sure the temp files are deleted
conf.delete_temporary_downloads_at_exit = True
# make sure the config and cache directories are not searched
monkeypatch.setenv('XDG_CONFIG_HOME', 'foo')
monkeypatch.delenv('XDG_CONFIG_HOME')
monkeypatch.setenv('XDG_CACHE_HOME', 'bar')
monkeypatch.delenv('XDG_CACHE_HOME')
monkeypatch.setattr(paths.set_temp_config, '_temp_path', None)
monkeypatch.setattr(paths.set_temp_cache, '_temp_path', None)
# make sure the _find_or_create_astropy_dir function fails as though the
# astropy dir could not be accessed
def osraiser(dirnm, linkto):
raise OSError
monkeypatch.setattr(paths, '_find_or_create_astropy_dir', osraiser)
with pytest.raises(OSError):
# make sure the config dir search fails
paths.get_cache_dir()
# first try with cache
with catch_warnings(CacheMissingWarning) as w:
fnout = data.download_file(TESTURL, cache=True)
assert os.path.isfile(fnout)
assert len(w) > 1
w1 = w.pop(0)
w2 = w.pop(0)
assert w1.category == CacheMissingWarning
assert 'Remote data cache could not be accessed' in w1.message.args[0]
assert w2.category == CacheMissingWarning
assert 'File downloaded to temporary location' in w2.message.args[0]
assert fnout == w2.message.args[1]
# clearing the cache should be a no-up that doesn't affect fnout
with catch_warnings(CacheMissingWarning) as w:
data.clear_download_cache(TESTURL)
assert os.path.isfile(fnout)
# now remove it so tests don't clutter up the temp dir this should get
# called at exit, anyway, but we do it here just to make sure it's working
# correctly
data._deltemps()
assert not os.path.isfile(fnout)
assert len(w) > 0
w3 = w.pop()
assert w3.category == data.CacheMissingWarning
assert 'Not clearing data cache - cache inacessable' in str(w3.message)
# now try with no cache
with catch_warnings(CacheMissingWarning) as w:
fnnocache = data.download_file(TESTURL, cache=False)
with open(fnnocache, 'rb') as page:
assert page.read().decode('utf-8').find('Astropy') > -1
# no warnings should be raise in fileobj because cache is unnecessary
assert len(w) == 0
# lockdir determined above as the *real* lockdir, not the temp one
assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
def test_data_noastropy_fallback(monkeypatch):
"""
Tests to make sure the default behavior when the cache directory can't
be located is correct
"""
from astropy.utils import data
from astropy.config import paths
# needed for testing the *real* lock at the end
lockdir = os.path.join(_get_download_cache_locs()[0], 'lock')
# better yet, set the configuration to make sure the temp files are deleted
conf.delete_temporary_downloads_at_exit = True
# make sure the config and cache directories are not searched
monkeypatch.setenv(str('XDG_CONFIG_HOME'), 'foo')
monkeypatch.delenv(str('XDG_CONFIG_HOME'))
monkeypatch.setenv(str('XDG_CACHE_HOME'), 'bar')
monkeypatch.delenv(str('XDG_CACHE_HOME'))
monkeypatch.setattr(paths.set_temp_config, '_temp_path', None)
monkeypatch.setattr(paths.set_temp_cache, '_temp_path', None)
# make sure the _find_or_create_astropy_dir function fails as though the
# astropy dir could not be accessed
def osraiser(dirnm, linkto):
raise OSError
monkeypatch.setattr(paths, '_find_or_create_astropy_dir', osraiser)
with pytest.raises(OSError):
# make sure the config dir search fails
paths.get_cache_dir()
# first try with cache
with catch_warnings(CacheMissingWarning) as w:
fnout = data.download_file(TESTURL, cache=True)
assert os.path.isfile(fnout)
assert len(w) > 1
w1 = w.pop(0)
w2 = w.pop(0)
assert w1.category == CacheMissingWarning
assert 'Remote data cache could not be accessed' in w1.message.args[0]
assert w2.category == CacheMissingWarning
assert 'File downloaded to temporary location' in w2.message.args[0]
assert fnout == w2.message.args[1]
# clearing the cache should be a no-up that doesn't affect fnout
with catch_warnings(CacheMissingWarning) as w:
data.clear_download_cache(TESTURL)
assert os.path.isfile(fnout)
# now remove it so tests don't clutter up the temp dir this should get
# called at exit, anyway, but we do it here just to make sure it's working
# correctly
data._deltemps()
assert not os.path.isfile(fnout)
assert len(w) > 0
w3 = w.pop()
assert w3.category == data.CacheMissingWarning
assert 'Not clearing data cache - cache inacessable' in str(w3.message)
# now try with no cache
with catch_warnings(CacheMissingWarning) as w:
fnnocache = data.download_file(TESTURL, cache=False)
with open(fnnocache, 'rb') as page:
assert page.read().decode('utf-8').find('Astropy') > -1
# no warnings should be raise in fileobj because cache is unnecessary
assert len(w) == 0
# lockdir determined above as the *real* lockdir, not the temp one
assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'