def test_name_resolve_cache(tmpdir): from astropy.utils.data import _get_download_cache_locs, get_cached_urls import shelve target_name = "castor" temp_cache_dir = str(tmpdir.mkdir('cache')) with paths.set_temp_cache(temp_cache_dir, delete=True): download_dir, urlmapfn = _get_download_cache_locs() with shelve.open(urlmapfn) as url2hash: assert len(url2hash) == 0 icrs1 = get_icrs_coordinates(target_name, cache=True) # This is a weak test: we just check to see that a url is added to the # cache! with shelve.open(urlmapfn) as url2hash: assert len(url2hash) == 1 url = get_cached_urls()[0] assert 'http://cdsweb.u-strasbg.fr/cgi-bin/nph-sesame/' in url # Try reloading coordinates, now should just reload cached data: with no_internet(): icrs2 = get_icrs_coordinates(target_name, cache=True) with shelve.open(urlmapfn) as url2hash: assert len(url2hash) == 1 assert u.allclose(icrs1.ra, icrs2.ra) assert u.allclose(icrs1.dec, icrs2.dec)
def test_download_cache(): from astropy.utils.data import download_file, clear_download_cache download_dir = _get_download_cache_locs()[0] # Download the test URL and make sure it exists, then clear just that # URL and make sure it got deleted. fnout = download_file(TESTURL, cache=True) assert os.path.isdir(download_dir) assert os.path.isfile(fnout) clear_download_cache(TESTURL) assert not os.path.exists(fnout) # Test issues raised in #4427 with clear_download_cache() without a URL, # followed by subsequent download. fnout = download_file(TESTURL, cache=True) assert os.path.isfile(fnout) clear_download_cache() assert not os.path.exists(fnout) assert not os.path.exists(download_dir) fnout = download_file(TESTURL, cache=True) assert os.path.isfile(fnout) # Clearing download cache succeeds even if the URL does not exist. clear_download_cache('http://this_was_never_downloaded_before.com') # Make sure lockdir was released lockdir = os.path.join(download_dir, 'lock') assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
def test_find_by_hash(): from astropy.utils.data import clear_download_cache with get_readable_fileobj(TESTURL, encoding="binary", cache=True) as page: hash = hashlib.md5(page.read()) hashstr = 'hash/' + hash.hexdigest() fnout = get_pkg_data_filename(hashstr) assert os.path.isfile(fnout) clear_download_cache(hashstr[5:]) assert not os.path.isfile(fnout) lockdir = os.path.join(_get_download_cache_locs()[0], 'lock') assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
def test_download_mirror_cache(): import pathlib import shelve from astropy.utils.data import _find_pkg_data_path, download_file, get_cached_urls main_url = pathlib.Path( _find_pkg_data_path(os.path.join('data', 'dataurl'))).as_uri() + '/' mirror_url = pathlib.Path( _find_pkg_data_path(os.path.join( 'data', 'dataurl_mirror'))).as_uri() + '/' # noqa main_file = main_url + 'index.html' mirror_file = mirror_url + 'index.html' # Temporarily change data.conf. # This also test https://github.com/astropy/astropy/pull/8163 because # urlopen() on a local dir URI also gives URLError. with conf.set_temp('dataurl', main_url): with conf.set_temp('dataurl_mirror', mirror_url): # "Download" files by rerouting URLs to local URIs. download_file(main_file, cache=True) download_file(mirror_file, cache=True) # Now test that download_file looks in mirror's cache before # download. # https://github.com/astropy/astropy/issues/6982 dldir, urlmapfn = _get_download_cache_locs() with shelve.open(urlmapfn) as url2hash: del url2hash[main_file] # Comparing hash makes sure they download the same file # but does not guarantee they were downloaded from the same URL. assert (download_file(main_file, cache=True) == download_file(mirror_file, cache=True)) # This has to be called after the last download to obtain # an accurate view of cached URLs. # This is to ensure that main_file was not re-downloaded # unnecessarily. # This test also tests for "assert TESTURL in get_cached_urls()". c_urls = get_cached_urls() assert (mirror_file in c_urls) and (main_file not in c_urls)
def IERS_A_in_cache(): """ Check if the IERS Bulletin A table is locally cached. """ url_key = iers.IERS_A_URL # The below code which accesses ``urlmapfn`` is stolen from # astropy.utils.data.download_file() try: dldir, urlmapfn = _get_download_cache_locs() except (IOError, OSError) as e: msg = 'Remote data cache could not be accessed due to ' estr = '' if len(e.args) < 1 else (': ' + str(e)) warnings.warn(CacheMissingWarning(msg + e.__class__.__name__ + estr)) return False with _open_shelve(urlmapfn, True) as url2hash: # TODO: try to figure out how to test this in the unicode case if str(url_key) in url2hash: return True return False
def save_fits(self, savepath, link_cache='hard'): """ Save a FITS file to savepath Parameters ---------- savepath : str The full path to a FITS filename, e.g. "file.fits", or "/path/to/file.fits". link_cache : 'hard', 'sym', or False Try to create a hard or symbolic link to the astropy cached file? If the system is unable to create a hardlink, the file will be copied to the target location. """ from warnings import warn self.get_fits() try: dldir, urlmapfn = aud._get_download_cache_locs() except (IOError, OSError) as e: msg = 'Remote data cache could not be accessed due to ' estr = '' if len(e.args) < 1 else (': ' + str(e)) warn(aud.CacheMissingWarning(msg + e.__class__.__name__ + estr)) with _open_shelve(urlmapfn, True) as url2hash: if str(self._target) in url2hash: target = url2hash[str(self._target)] else: raise IOError("Cached file not found / does not exist.") if link_cache == 'hard': try: os.link(target, savepath) except (IOError, OSError, AttributeError): shutil.copy(target, savepath) elif link_cache == 'sym': try: os.symlink(target, savepath) except AttributeError: raise OSError('Creating symlinks is not possible on this OS.') else: shutil.copy(target, savepath)
def save_fits(self, savepath, link_cache='hard'): """ Save a FITS file to savepath Parameters ---------- savepath : str The full path to a FITS filename, e.g. "file.fits", or "/path/to/file.fits". link_cache : 'hard', 'sym', or False Try to create a hard or symbolic link to the astropy cached file? If the system is unable to create a hardlink, the file will be copied to the target location. """ from warnings import warn self.get_fits() try: dldir, urlmapfn = aud._get_download_cache_locs() except (IOError, OSError) as e: msg = 'Remote data cache could not be accessed due to ' estr = '' if len(e.args) < 1 else (': ' + str(e)) warn(aud.CacheMissingWarning(msg + e.__class__.__name__ + estr)) with aud._open_shelve(urlmapfn, True) as url2hash: if str(self._target) in url2hash: target = url2hash[str(self._target)] else: raise IOError("Cached file not found / does not exist.") if link_cache == 'hard': try: os.link(target, savepath) except (IOError, OSError, AttributeError) as e: shutil.copy(target, savepath) elif link_cache == 'sym': try: os.symlink(target, savepath) except AttributeError: raise OSError('Creating symlinks is not possible on this OS.') else: shutil.copy(target, savepath)
def test_download_mirror_cache(): import pathlib import shelve from astropy.utils.data import _find_pkg_data_path, download_file, get_cached_urls main_url = pathlib.Path( _find_pkg_data_path(os.path.join('data', 'dataurl'))).as_uri() + '/' mirror_url = pathlib.Path( _find_pkg_data_path(os.path.join('data', 'dataurl_mirror'))).as_uri() + '/' # noqa main_file = main_url + 'index.html' mirror_file = mirror_url + 'index.html' # Temporarily change data.conf. # This also test https://github.com/astropy/astropy/pull/8163 because # urlopen() on a local dir URI also gives URLError. with conf.set_temp('dataurl', main_url): with conf.set_temp('dataurl_mirror', mirror_url): # "Download" files by rerouting URLs to local URIs. download_file(main_file, cache=True) download_file(mirror_file, cache=True) # Now test that download_file looks in mirror's cache before # download. # https://github.com/astropy/astropy/issues/6982 dldir, urlmapfn = _get_download_cache_locs() with shelve.open(urlmapfn) as url2hash: del url2hash[main_file] # Comparing hash makes sure they download the same file # but does not guarantee they were downloaded from the same URL. assert (download_file(main_file, cache=True) == download_file(mirror_file, cache=True)) # This has to be called after the last download to obtain # an accurate view of cached URLs. # This is to ensure that main_file was not re-downloaded # unnecessarily. # This test also tests for "assert TESTURL in get_cached_urls()". c_urls = get_cached_urls() assert (mirror_file in c_urls) and (main_file not in c_urls)
def test_data_noastropy_fallback(monkeypatch): """ Tests to make sure the default behavior when the cache directory can't be located is correct """ from astropy.utils import data from astropy.config import paths # needed for testing the *real* lock at the end lockdir = os.path.join(_get_download_cache_locs()[0], 'lock') # better yet, set the configuration to make sure the temp files are deleted conf.delete_temporary_downloads_at_exit = True # make sure the config and cache directories are not searched monkeypatch.setenv('XDG_CONFIG_HOME', 'foo') monkeypatch.delenv('XDG_CONFIG_HOME') monkeypatch.setenv('XDG_CACHE_HOME', 'bar') monkeypatch.delenv('XDG_CACHE_HOME') monkeypatch.setattr(paths.set_temp_config, '_temp_path', None) monkeypatch.setattr(paths.set_temp_cache, '_temp_path', None) # make sure the _find_or_create_astropy_dir function fails as though the # astropy dir could not be accessed def osraiser(dirnm, linkto): raise OSError monkeypatch.setattr(paths, '_find_or_create_astropy_dir', osraiser) with pytest.raises(OSError): # make sure the config dir search fails paths.get_cache_dir() # first try with cache with catch_warnings(CacheMissingWarning) as w: fnout = data.download_file(TESTURL, cache=True) assert os.path.isfile(fnout) assert len(w) > 1 w1 = w.pop(0) w2 = w.pop(0) assert w1.category == CacheMissingWarning assert 'Remote data cache could not be accessed' in w1.message.args[0] assert w2.category == CacheMissingWarning assert 'File downloaded to temporary location' in w2.message.args[0] assert fnout == w2.message.args[1] # clearing the cache should be a no-up that doesn't affect fnout with catch_warnings(CacheMissingWarning) as w: data.clear_download_cache(TESTURL) assert os.path.isfile(fnout) # now remove it so tests don't clutter up the temp dir this should get # called at exit, anyway, but we do it here just to make sure it's working # correctly data._deltemps() assert not os.path.isfile(fnout) assert len(w) > 0 w3 = w.pop() assert w3.category == data.CacheMissingWarning assert 'Not clearing data cache - cache inacessable' in str(w3.message) # now try with no cache with catch_warnings(CacheMissingWarning) as w: fnnocache = data.download_file(TESTURL, cache=False) with open(fnnocache, 'rb') as page: assert page.read().decode('utf-8').find('Astropy') > -1 # no warnings should be raise in fileobj because cache is unnecessary assert len(w) == 0 # lockdir determined above as the *real* lockdir, not the temp one assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
def test_data_noastropy_fallback(monkeypatch): """ Tests to make sure the default behavior when the cache directory can't be located is correct """ from astropy.utils import data from astropy.config import paths # needed for testing the *real* lock at the end lockdir = os.path.join(_get_download_cache_locs()[0], 'lock') # better yet, set the configuration to make sure the temp files are deleted conf.delete_temporary_downloads_at_exit = True # make sure the config and cache directories are not searched monkeypatch.setenv(str('XDG_CONFIG_HOME'), 'foo') monkeypatch.delenv(str('XDG_CONFIG_HOME')) monkeypatch.setenv(str('XDG_CACHE_HOME'), 'bar') monkeypatch.delenv(str('XDG_CACHE_HOME')) monkeypatch.setattr(paths.set_temp_config, '_temp_path', None) monkeypatch.setattr(paths.set_temp_cache, '_temp_path', None) # make sure the _find_or_create_astropy_dir function fails as though the # astropy dir could not be accessed def osraiser(dirnm, linkto): raise OSError monkeypatch.setattr(paths, '_find_or_create_astropy_dir', osraiser) with pytest.raises(OSError): # make sure the config dir search fails paths.get_cache_dir() # first try with cache with catch_warnings(CacheMissingWarning) as w: fnout = data.download_file(TESTURL, cache=True) assert os.path.isfile(fnout) assert len(w) > 1 w1 = w.pop(0) w2 = w.pop(0) assert w1.category == CacheMissingWarning assert 'Remote data cache could not be accessed' in w1.message.args[0] assert w2.category == CacheMissingWarning assert 'File downloaded to temporary location' in w2.message.args[0] assert fnout == w2.message.args[1] # clearing the cache should be a no-up that doesn't affect fnout with catch_warnings(CacheMissingWarning) as w: data.clear_download_cache(TESTURL) assert os.path.isfile(fnout) # now remove it so tests don't clutter up the temp dir this should get # called at exit, anyway, but we do it here just to make sure it's working # correctly data._deltemps() assert not os.path.isfile(fnout) assert len(w) > 0 w3 = w.pop() assert w3.category == data.CacheMissingWarning assert 'Not clearing data cache - cache inacessable' in str(w3.message) # now try with no cache with catch_warnings(CacheMissingWarning) as w: fnnocache = data.download_file(TESTURL, cache=False) with open(fnnocache, 'rb') as page: assert page.read().decode('utf-8').find('Astropy') > -1 # no warnings should be raise in fileobj because cache is unnecessary assert len(w) == 0 # lockdir determined above as the *real* lockdir, not the temp one assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'