Exemplo n.º 1
0
def test_name_resolve_cache(tmpdir):
    from astropy.utils.data import _get_download_cache_locs, get_cached_urls
    import shelve

    target_name = "castor"

    temp_cache_dir = str(tmpdir.mkdir('cache'))
    with paths.set_temp_cache(temp_cache_dir, delete=True):
        download_dir, urlmapfn = _get_download_cache_locs()

        with shelve.open(urlmapfn) as url2hash:
            assert len(url2hash) == 0

        icrs1 = get_icrs_coordinates(target_name, cache=True)

        # This is a weak test: we just check to see that a url is added to the
        #  cache!
        with shelve.open(urlmapfn) as url2hash:
            assert len(url2hash) == 1
            url = get_cached_urls()[0]
            assert 'http://cdsweb.u-strasbg.fr/cgi-bin/nph-sesame/' in url

        # Try reloading coordinates, now should just reload cached data:
        with no_internet():
            icrs2 = get_icrs_coordinates(target_name, cache=True)

        with shelve.open(urlmapfn) as url2hash:
            assert len(url2hash) == 1

        assert u.allclose(icrs1.ra, icrs2.ra)
        assert u.allclose(icrs1.dec, icrs2.dec)
Exemplo n.º 2
0
def test_download_cache():

    from astropy.utils.data import download_file, clear_download_cache

    download_dir = _get_download_cache_locs()[0]

    # Download the test URL and make sure it exists, then clear just that
    # URL and make sure it got deleted.
    fnout = download_file(TESTURL, cache=True)
    assert os.path.isdir(download_dir)
    assert os.path.isfile(fnout)
    clear_download_cache(TESTURL)
    assert not os.path.exists(fnout)

    # Test issues raised in #4427 with clear_download_cache() without a URL,
    # followed by subsequent download.
    fnout = download_file(TESTURL, cache=True)
    assert os.path.isfile(fnout)
    clear_download_cache()
    assert not os.path.exists(fnout)
    assert not os.path.exists(download_dir)
    fnout = download_file(TESTURL, cache=True)
    assert os.path.isfile(fnout)

    # Clearing download cache succeeds even if the URL does not exist.
    clear_download_cache('http://this_was_never_downloaded_before.com')

    # Make sure lockdir was released
    lockdir = os.path.join(download_dir, 'lock')
    assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
Exemplo n.º 3
0
def test_download_cache():

    from astropy.utils.data import download_file, clear_download_cache

    download_dir = _get_download_cache_locs()[0]

    # Download the test URL and make sure it exists, then clear just that
    # URL and make sure it got deleted.
    fnout = download_file(TESTURL, cache=True)
    assert os.path.isdir(download_dir)
    assert os.path.isfile(fnout)
    clear_download_cache(TESTURL)
    assert not os.path.exists(fnout)

    # Test issues raised in #4427 with clear_download_cache() without a URL,
    # followed by subsequent download.
    fnout = download_file(TESTURL, cache=True)
    assert os.path.isfile(fnout)
    clear_download_cache()
    assert not os.path.exists(fnout)
    assert not os.path.exists(download_dir)
    fnout = download_file(TESTURL, cache=True)
    assert os.path.isfile(fnout)

    # Clearing download cache succeeds even if the URL does not exist.
    clear_download_cache('http://this_was_never_downloaded_before.com')

    # Make sure lockdir was released
    lockdir = os.path.join(download_dir, 'lock')
    assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
Exemplo n.º 4
0
def test_find_by_hash():

    from astropy.utils.data import clear_download_cache

    with get_readable_fileobj(TESTURL, encoding="binary", cache=True) as page:
        hash = hashlib.md5(page.read())

    hashstr = 'hash/' + hash.hexdigest()

    fnout = get_pkg_data_filename(hashstr)
    assert os.path.isfile(fnout)
    clear_download_cache(hashstr[5:])
    assert not os.path.isfile(fnout)

    lockdir = os.path.join(_get_download_cache_locs()[0], 'lock')
    assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
Exemplo n.º 5
0
def test_find_by_hash():

    from astropy.utils.data import clear_download_cache

    with get_readable_fileobj(TESTURL, encoding="binary", cache=True) as page:
        hash = hashlib.md5(page.read())

    hashstr = 'hash/' + hash.hexdigest()

    fnout = get_pkg_data_filename(hashstr)
    assert os.path.isfile(fnout)
    clear_download_cache(hashstr[5:])
    assert not os.path.isfile(fnout)

    lockdir = os.path.join(_get_download_cache_locs()[0], 'lock')
    assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
Exemplo n.º 6
0
def test_download_mirror_cache():
    import pathlib
    import shelve
    from astropy.utils.data import _find_pkg_data_path, download_file, get_cached_urls

    main_url = pathlib.Path(
        _find_pkg_data_path(os.path.join('data', 'dataurl'))).as_uri() + '/'
    mirror_url = pathlib.Path(
        _find_pkg_data_path(os.path.join(
            'data', 'dataurl_mirror'))).as_uri() + '/'  # noqa

    main_file = main_url + 'index.html'
    mirror_file = mirror_url + 'index.html'

    # Temporarily change data.conf.
    # This also test https://github.com/astropy/astropy/pull/8163 because
    # urlopen() on a local dir URI also gives URLError.
    with conf.set_temp('dataurl', main_url):
        with conf.set_temp('dataurl_mirror', mirror_url):

            # "Download" files by rerouting URLs to local URIs.
            download_file(main_file, cache=True)
            download_file(mirror_file, cache=True)

            # Now test that download_file looks in mirror's cache before
            # download.
            # https://github.com/astropy/astropy/issues/6982
            dldir, urlmapfn = _get_download_cache_locs()
            with shelve.open(urlmapfn) as url2hash:
                del url2hash[main_file]

            # Comparing hash makes sure they download the same file
            # but does not guarantee they were downloaded from the same URL.
            assert (download_file(main_file,
                                  cache=True) == download_file(mirror_file,
                                                               cache=True))

            # This has to be called after the last download to obtain
            # an accurate view of cached URLs.
            # This is to ensure that main_file was not re-downloaded
            # unnecessarily.
            # This test also tests for "assert TESTURL in get_cached_urls()".
            c_urls = get_cached_urls()
            assert (mirror_file in c_urls) and (main_file not in c_urls)
Exemplo n.º 7
0
def IERS_A_in_cache():
    """
    Check if the IERS Bulletin A table is locally cached.
    """
    url_key = iers.IERS_A_URL
    # The below code which accesses ``urlmapfn`` is stolen from
    # astropy.utils.data.download_file()
    try:
        dldir, urlmapfn = _get_download_cache_locs()
    except (IOError, OSError) as e:
        msg = 'Remote data cache could not be accessed due to '
        estr = '' if len(e.args) < 1 else (': ' + str(e))
        warnings.warn(CacheMissingWarning(msg + e.__class__.__name__ + estr))
        return False
    with _open_shelve(urlmapfn, True) as url2hash:
        # TODO: try to figure out how to test this in the unicode case
        if str(url_key) in url2hash:
            return True
    return False
Exemplo n.º 8
0
    def save_fits(self, savepath, link_cache='hard'):
        """
        Save a FITS file to savepath

        Parameters
        ----------
        savepath : str
            The full path to a FITS filename, e.g. "file.fits", or
            "/path/to/file.fits".
        link_cache : 'hard', 'sym', or False
            Try to create a hard or symbolic link to the astropy cached file?
            If the system is unable to create a hardlink, the file will be
            copied to the target location.
        """
        from warnings import warn

        self.get_fits()

        try:
            dldir, urlmapfn = aud._get_download_cache_locs()
        except (IOError, OSError) as e:
            msg = 'Remote data cache could not be accessed due to '
            estr = '' if len(e.args) < 1 else (': ' + str(e))
            warn(aud.CacheMissingWarning(msg + e.__class__.__name__ + estr))

        with _open_shelve(urlmapfn, True) as url2hash:
            if str(self._target) in url2hash:
                target = url2hash[str(self._target)]
            else:
                raise IOError("Cached file not found / does not exist.")

        if link_cache == 'hard':
            try:
                os.link(target, savepath)
            except (IOError, OSError, AttributeError):
                shutil.copy(target, savepath)
        elif link_cache == 'sym':
            try:
                os.symlink(target, savepath)
            except AttributeError:
                raise OSError('Creating symlinks is not possible on this OS.')
        else:
            shutil.copy(target, savepath)
Exemplo n.º 9
0
def IERS_A_in_cache():
    """
    Check if the IERS Bulletin A table is locally cached.
    """
    url_key = iers.IERS_A_URL
    # The below code which accesses ``urlmapfn`` is stolen from
    # astropy.utils.data.download_file()
    try:
        dldir, urlmapfn = _get_download_cache_locs()
    except (IOError, OSError) as e:
        msg = 'Remote data cache could not be accessed due to '
        estr = '' if len(e.args) < 1 else (': ' + str(e))
        warnings.warn(CacheMissingWarning(msg + e.__class__.__name__ + estr))
        return False
    with _open_shelve(urlmapfn, True) as url2hash:
        # TODO: try to figure out how to test this in the unicode case
        if str(url_key) in url2hash:
            return True
    return False
Exemplo n.º 10
0
    def save_fits(self, savepath, link_cache='hard'):
        """
        Save a FITS file to savepath

        Parameters
        ----------
        savepath : str
            The full path to a FITS filename, e.g. "file.fits", or
            "/path/to/file.fits".
        link_cache : 'hard', 'sym', or False
            Try to create a hard or symbolic link to the astropy cached file?
            If the system is unable to create a hardlink, the file will be
            copied to the target location.
        """
        from warnings import warn

        self.get_fits()

        try:
            dldir, urlmapfn = aud._get_download_cache_locs()
        except (IOError, OSError) as e:
            msg = 'Remote data cache could not be accessed due to '
            estr = '' if len(e.args) < 1 else (': ' + str(e))
            warn(aud.CacheMissingWarning(msg + e.__class__.__name__ + estr))

        with aud._open_shelve(urlmapfn, True) as url2hash:
            if str(self._target) in url2hash:
                target = url2hash[str(self._target)]
            else:
                raise IOError("Cached file not found / does not exist.")

        if link_cache == 'hard':
            try:
                os.link(target, savepath)
            except (IOError, OSError, AttributeError) as e:
                shutil.copy(target, savepath)
        elif link_cache == 'sym':
            try:
                os.symlink(target, savepath)
            except AttributeError:
                raise OSError('Creating symlinks is not possible on this OS.')
        else:
            shutil.copy(target, savepath)
Exemplo n.º 11
0
def test_download_mirror_cache():
    import pathlib
    import shelve
    from astropy.utils.data import _find_pkg_data_path, download_file, get_cached_urls

    main_url = pathlib.Path(
        _find_pkg_data_path(os.path.join('data', 'dataurl'))).as_uri() + '/'
    mirror_url = pathlib.Path(
        _find_pkg_data_path(os.path.join('data', 'dataurl_mirror'))).as_uri() + '/'  # noqa

    main_file = main_url + 'index.html'
    mirror_file = mirror_url + 'index.html'

    # Temporarily change data.conf.
    # This also test https://github.com/astropy/astropy/pull/8163 because
    # urlopen() on a local dir URI also gives URLError.
    with conf.set_temp('dataurl', main_url):
        with conf.set_temp('dataurl_mirror', mirror_url):

            # "Download" files by rerouting URLs to local URIs.
            download_file(main_file, cache=True)
            download_file(mirror_file, cache=True)

            # Now test that download_file looks in mirror's cache before
            # download.
            # https://github.com/astropy/astropy/issues/6982
            dldir, urlmapfn = _get_download_cache_locs()
            with shelve.open(urlmapfn) as url2hash:
                del url2hash[main_file]

            # Comparing hash makes sure they download the same file
            # but does not guarantee they were downloaded from the same URL.
            assert (download_file(main_file, cache=True) ==
                    download_file(mirror_file, cache=True))

            # This has to be called after the last download to obtain
            # an accurate view of cached URLs.
            # This is to ensure that main_file was not re-downloaded
            # unnecessarily.
            # This test also tests for "assert TESTURL in get_cached_urls()".
            c_urls = get_cached_urls()
            assert (mirror_file in c_urls) and (main_file not in c_urls)
Exemplo n.º 12
0
def test_data_noastropy_fallback(monkeypatch):
    """
    Tests to make sure the default behavior when the cache directory can't
    be located is correct
    """

    from astropy.utils import data
    from astropy.config import paths

    # needed for testing the *real* lock at the end
    lockdir = os.path.join(_get_download_cache_locs()[0], 'lock')

    # better yet, set the configuration to make sure the temp files are deleted
    conf.delete_temporary_downloads_at_exit = True

    # make sure the config and cache directories are not searched
    monkeypatch.setenv('XDG_CONFIG_HOME', 'foo')
    monkeypatch.delenv('XDG_CONFIG_HOME')
    monkeypatch.setenv('XDG_CACHE_HOME', 'bar')
    monkeypatch.delenv('XDG_CACHE_HOME')

    monkeypatch.setattr(paths.set_temp_config, '_temp_path', None)
    monkeypatch.setattr(paths.set_temp_cache, '_temp_path', None)

    # make sure the _find_or_create_astropy_dir function fails as though the
    # astropy dir could not be accessed
    def osraiser(dirnm, linkto):
        raise OSError
    monkeypatch.setattr(paths, '_find_or_create_astropy_dir', osraiser)

    with pytest.raises(OSError):
        # make sure the config dir search fails
        paths.get_cache_dir()

    # first try with cache
    with catch_warnings(CacheMissingWarning) as w:
        fnout = data.download_file(TESTURL, cache=True)

    assert os.path.isfile(fnout)

    assert len(w) > 1

    w1 = w.pop(0)
    w2 = w.pop(0)

    assert w1.category == CacheMissingWarning
    assert 'Remote data cache could not be accessed' in w1.message.args[0]
    assert w2.category == CacheMissingWarning
    assert 'File downloaded to temporary location' in w2.message.args[0]
    assert fnout == w2.message.args[1]

    # clearing the cache should be a no-up that doesn't affect fnout
    with catch_warnings(CacheMissingWarning) as w:
        data.clear_download_cache(TESTURL)
    assert os.path.isfile(fnout)

    # now remove it so tests don't clutter up the temp dir this should get
    # called at exit, anyway, but we do it here just to make sure it's working
    # correctly
    data._deltemps()
    assert not os.path.isfile(fnout)

    assert len(w) > 0
    w3 = w.pop()

    assert w3.category == data.CacheMissingWarning
    assert 'Not clearing data cache - cache inacessable' in str(w3.message)

    # now try with no cache
    with catch_warnings(CacheMissingWarning) as w:
        fnnocache = data.download_file(TESTURL, cache=False)
    with open(fnnocache, 'rb') as page:
        assert page.read().decode('utf-8').find('Astropy') > -1

    # no warnings should be raise in fileobj because cache is unnecessary
    assert len(w) == 0

    # lockdir determined above as the *real* lockdir, not the temp one
    assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'
Exemplo n.º 13
0
def test_data_noastropy_fallback(monkeypatch):
    """
    Tests to make sure the default behavior when the cache directory can't
    be located is correct
    """

    from astropy.utils import data
    from astropy.config import paths

    # needed for testing the *real* lock at the end
    lockdir = os.path.join(_get_download_cache_locs()[0], 'lock')

    # better yet, set the configuration to make sure the temp files are deleted
    conf.delete_temporary_downloads_at_exit = True

    # make sure the config and cache directories are not searched
    monkeypatch.setenv(str('XDG_CONFIG_HOME'), 'foo')
    monkeypatch.delenv(str('XDG_CONFIG_HOME'))
    monkeypatch.setenv(str('XDG_CACHE_HOME'), 'bar')
    monkeypatch.delenv(str('XDG_CACHE_HOME'))

    monkeypatch.setattr(paths.set_temp_config, '_temp_path', None)
    monkeypatch.setattr(paths.set_temp_cache, '_temp_path', None)

    # make sure the _find_or_create_astropy_dir function fails as though the
    # astropy dir could not be accessed
    def osraiser(dirnm, linkto):
        raise OSError
    monkeypatch.setattr(paths, '_find_or_create_astropy_dir', osraiser)

    with pytest.raises(OSError):
        # make sure the config dir search fails
        paths.get_cache_dir()

    # first try with cache
    with catch_warnings(CacheMissingWarning) as w:
        fnout = data.download_file(TESTURL, cache=True)

    assert os.path.isfile(fnout)

    assert len(w) > 1

    w1 = w.pop(0)
    w2 = w.pop(0)

    assert w1.category == CacheMissingWarning
    assert 'Remote data cache could not be accessed' in w1.message.args[0]
    assert w2.category == CacheMissingWarning
    assert 'File downloaded to temporary location' in w2.message.args[0]
    assert fnout == w2.message.args[1]

    # clearing the cache should be a no-up that doesn't affect fnout
    with catch_warnings(CacheMissingWarning) as w:
        data.clear_download_cache(TESTURL)
    assert os.path.isfile(fnout)

    # now remove it so tests don't clutter up the temp dir this should get
    # called at exit, anyway, but we do it here just to make sure it's working
    # correctly
    data._deltemps()
    assert not os.path.isfile(fnout)

    assert len(w) > 0
    w3 = w.pop()

    assert w3.category == data.CacheMissingWarning
    assert 'Not clearing data cache - cache inacessable' in str(w3.message)

    # now try with no cache
    with catch_warnings(CacheMissingWarning) as w:
        fnnocache = data.download_file(TESTURL, cache=False)
    with open(fnnocache, 'rb') as page:
        assert page.read().decode('utf-8').find('Astropy') > -1

    # no warnings should be raise in fileobj because cache is unnecessary
    assert len(w) == 0

    # lockdir determined above as the *real* lockdir, not the temp one
    assert not os.path.isdir(lockdir), 'Cache dir lock was not released!'