def test_close(self):
cache = mock.Mock(spec=DictCache)
sess = Session()
sess.mount('http://', CacheControlAdapter(cache))
sess.close()
assert cache.close.called
def session(self):
session = requests.session()
if self.request_cache:
try:
from cachecontrol.adapter import CacheControlAdapter
adapter = CacheControlAdapter(
cache=self.application.request_cache
if self.application else self.request_cache,
heuristic=self.cache_heuristic,
cacheable_methods=("GET", ),
)
if self.api:
# we will only cache GET requests to the /boxes path
session.mount(urljoin(self.api, paths.BOXES), adapter)
else:
logger.warning(
"No API is set. "
"Mounting cache adapter for whole http(s):// schema")
for schema in ("http://", "https://"):
session.mount(schema, adapter)
except ImportError:
logger.warning(
"CacheControl is not available. Cannot use cache.")
return session
def sess(self, url):
self.url = url
self.cache = DictCache()
sess = Session()
sess.mount(
"http://",
CacheControlAdapter(self.cache, serializer=NullSerializer()))
return sess
def __init__(self,
internal=False,
circuit_breaker=None,
good_codes=(),
bad_codes=range(400, 600),
cache=None,
timeout=None):
Session.__init__(self)
self.internal = internal
self.circuit_breaker = circuit_breaker or DEFAULT_CIRCUIT_BREAKER
self.good_codes = good_codes
self.bad_codes = bad_codes
if cache is None:
cache = DEFAULT_HTTP_CACHE
self.mount('http://', CacheControlAdapter(cache=cache))
self.mount('https://', CacheControlAdapter(cache=cache))
self.timeout = timeout or self.DEFAULT_TIMEOUT
def sess(self, chunking_server):
self.url = chunking_server.base_url
self.cache = DictCache()
sess = Session()
sess.mount(
'http://',
CacheControlAdapter(self.cache, serializer=NullSerializer()),
)
return sess
def sess(self, server):
self.url = server.application_url
self.cache = DictCache()
sess = Session()
sess.mount(
'http://',
CacheControlAdapter(self.cache, serializer=NullSerializer()),
)
return sess
def get_session():
adapter = CacheControlAdapter(
DictCache(), cache_etags=True, serializer=None, heuristic=None
)
sess = requests.Session()
sess.mount("http://", adapter)
sess.mount("https://", adapter)
sess.cache_controller = adapter.controller
return sess
def get_session():
CACHE_FOLDER.mkdir(exist_ok=True)
cache = FileCache(str(CACHE_FOLDER), forever=True)
cache.set("foo", b"bar")
assert cache.get("foo") == b"bar"
session = RateLimitingSession()
# session.headers.update({"x-api-key": "something-something-darkside"})
session.mount(
"https://www.metlink.org.nz/",
CacheControlAdapter(heuristic=BetterExpiresAfter(days=7), cache=cache),
)
session.mount(
METLINK_API_URL_PREFIX,
CacheControlAdapter(heuristic=BetterExpiresAfter(days=1), cache=cache),
)
session.mount(
METLINK_API_URL_PREFIX + "ServiceLocation/",
CacheControlAdapter(heuristic=BetterExpiresAfter(seconds=90),
cache=cache),
)
return session
def make_session(scraper):
""" Instantiate a session with the desired configuration parameters,
including the cache policy. """
cache_path = os.path.join(scraper.config.data_path, 'cache')
cache_policy = scraper.config.cache_policy
cache_policy = cache_policy.lower().strip()
session = ScraperSession()
session.scraper = scraper
session.cache_policy = cache_policy
adapter = CacheControlAdapter(FileCache(cache_path),
cache_etags=True,
controller_class=PolicyCacheController)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def __init__(self, github_token, repo, args):
github = github3.login(token=github_token)
github.session.mount('https://api.github.com', CacheControlAdapter())
self.repo = github.repository(repo['owner'], repo['name'])
self.upstream_repo = github.repository('freeipa', 'freeipa')
self.args = args
def CacheControl(sess, cache=None, cache_etags=True):
cache = cache or DictCache()
adapter = CacheControlAdapter(cache, cache_etags=cache_etags)
sess.mount('http://', adapter)
return sess
import time
import requests
from cachecontrol.adapter import CacheControlAdapter
from cachecontrol.caches import FileCache
from cachecontrol.heuristics import ExpiresAfter, LastModified
from .utils import BASE
adapter = CacheControlAdapter(heuristic=ExpiresAfter(days=1))
current_time = time.time
CACHE_FOLDER = BASE / ".web_cache"
RATE_LIMIT_SECONDS = 1
RATE_LIMIT_REQUESTS = 9999
METLINK_API_URL_PREFIX = "https://www.metlink.org.nz/api/v1/"
class BetterExpiresAfter(ExpiresAfter):
def update_headers(self, response):
if response.status not in LastModified.cacheable_by_default_statuses:
return {}
return super().update_headers(response)
class DebugFileCache(FileCache):
def get(self, key):
print(f"get({key!r})")
return super().get(key)
def use_adapter():
print('Using adapter')
sess = Session()
sess.mount('http://', CacheControlAdapter())
return sess
def use_adapter():
print("Using adapter")
sess = Session()
sess.mount("http://", CacheControlAdapter())
return sess
__doc__ = """iscacheable - Determine if a URL is URL cacheable or not
Usage:
iscacheable <url>
iscacheable -h | --help
iscacheable --version
Options:
-h --help Show this help information.
--version Show version.
"""
sess = CacheControl(requests.session())
adapter = CacheControlAdapter(
cache_etags=True, cacheable_methods=('GET', 'HEAD'))
sess.mount('http://', adapter)
sess.mount('https://', adapter)
def determineCacheability(url):
resp = sess.head(url, allow_redirects=True)
adapter.controller.cache_response(resp.request, resp.raw)
if adapter.controller.cached_request(resp.request):
return True
return False
def main():
args = docopt(__doc__, version=VERSION) # Raises SystemExit.