def __init__(self):
self.settings = EsiSettings.getInstance()
self.server_base: ApiBase = supported_servers[self.settings.get("server")]
# session request stuff
self._session = Session()
self._basicHeaders = {
'Accept': 'application/json',
'User-Agent': (
'pyfa v{}'.format(config.version)
)
}
self._session.headers.update(self._basicHeaders)
self._session.proxies = NetworkSettings.getInstance().getProxySettingsInRequestsFormat()
# Set up cached session. This is only used for SSO meta data for now, but can be expanded to actually handle
# various ESI caching (using ETag, for example) in the future
cached_session = CachedSession(
os.path.join(config.savePath, config.ESI_CACHE),
backend="sqlite",
cache_control=True, # Use Cache-Control headers for expiration, if available
expire_after=timedelta(days=1), # Otherwise expire responses after one day
stale_if_error=True, # In case of request errors, use stale cache data if possible
)
cached_session.headers.update(self._basicHeaders)
cached_session.proxies = NetworkSettings.getInstance().getProxySettingsInRequestsFormat()
meta_call = cached_session.get("https://%s/.well-known/oauth-authorization-server" % self.server_base.sso)
meta_call.raise_for_status()
self.server_meta = meta_call.json()
jwks_call = cached_session.get(self.server_meta["jwks_uri"])
jwks_call.raise_for_status()
self.jwks = jwks_call.json()
def __init__(self):
super(AsteroidsSkill, self).__init__(name="AsteroidsSkill")
if "nasa_key" not in self.settings:
self.settings["nasa_key"] = "DEMO_KEY"
_expire_after = timedelta(hours=1)
self._session = CachedSession(backend='memory',
expire_after=_expire_after)
def test_return_old_data_on_error(self, datetime_mock):
datetime_mock.utcnow.return_value = datetime.utcnow()
expire_after = 100
url = httpbin("get")
s = CachedSession(CACHE_NAME, CACHE_BACKEND, old_data_on_error=True, expire_after=expire_after)
header = "X-Tst"
def get(n):
return s.get(url, headers={header: n}).json()["headers"][header]
get("expired")
self.assertEquals(get("2"), "expired")
datetime_mock.utcnow.return_value = datetime.utcnow() + timedelta(seconds=expire_after * 2)
with mock.patch.object(s.cache, "save_response", side_effect=Exception):
self.assertEquals(get("3"), "expired")
with mock.patch("requests_cache.core.OriginalSession.send") as send_mock:
resp_mock = requests.Response()
request = requests.Request("GET", url)
resp_mock.request = request.prepare()
resp_mock.status_code = 400
resp_mock._content = '{"other": "content"}'
send_mock.return_value = resp_mock
self.assertEquals(get("3"), "expired")
resp_mock.status_code = 200
self.assertIs(s.get(url).content, resp_mock.content)
# default behaviour
datetime_mock.return_value = datetime.utcnow() + timedelta(seconds=expire_after * 2)
s = CachedSession(CACHE_NAME, CACHE_BACKEND, old_data_on_error=False, expire_after=100)
with mock.patch.object(s.cache, "save_response", side_effect=Exception):
with self.assertRaises(Exception):
s.get(url)
def __init__(self,
source=None,
filter=None,
reverse_geocode=False,
progressbar=False,
quick_mode=False,
dry_run=False):
self.source = source
self.reverse_geocode = reverse_geocode
self.dry_run = dry_run
self.progressbar = progressbar
self.filter = filter
# Quick mode only imports the first few datasets to speed things up.
self.quick_mode = quick_mode
# Cache responses from the luftdaten.info API for five minutes.
# TODO: Make backend configurable.
# Configure User-Agent string.
user_agent = APP_NAME + '/' + APP_VERSION
# Configure cached requests session.
self.session = CachedSession(cache_name='api.luftdaten.info',
backend='redis',
expire_after=300,
user_agent=user_agent)
# Probe Redis for availability.
try:
self.session.cache.responses.get('test')
except redis.exceptions.ConnectionError as ex:
log.error('Unable to connect to Redis: %s', ex)
sys.exit(2)
def __init__(self):
super().__init__("XKCDSkill")
if not self.settings.get("idle_random"):
self.settings["idle_random"] = True
self.session = CachedSession(backend='memory',
expire_after=timedelta(hours=6))
self.current_comic = 0
def __init__(
self,
client_id,
client_secret,
cache_name: str,
token=None,
token_secret=None,
redirect_uri=None,
rsa_key=None,
verifier=None,
signature_method=SIGNATURE_HMAC_SHA1,
signature_type=SIGNATURE_TYPE_HEADER,
force_include_body=False,
backend='redis',
expire_after=300,
**kwargs,
):
CachedSession.__init__(self, cache_name, backend, expire_after)
OAuth1Client.__init__(
self,
session=self,
client_id=client_id,
client_secret=client_secret,
token=token,
token_secret=token_secret,
redirect_uri=redirect_uri,
rsa_key=rsa_key,
verifier=verifier,
signature_method=signature_method,
signature_type=signature_type,
force_include_body=force_include_body,
**kwargs,
)
def main():
session = CachedSession('example_cache', backend='sqlite')
# By default, cached responses never expire
response = session.get('https://httpbin.org/get')
assert not response.from_cache
response = session.get('https://httpbin.org/get')
assert response.from_cache
assert not response.expires
# We can set default expiration for the session using expire_after
session = CachedSession('example_cache', backend='sqlite', expire_after=60)
session.cache.clear()
response = session.get('https://httpbin.org/get')
response = session.get('https://httpbin.org/get')
print('Expiration time:', response.expires)
# This can also be overridden for individual requests
session.cache.clear()
response = session.get('https://httpbin.org/get', expire_after=1)
response = session.get('https://httpbin.org/get')
assert response.from_cache
print('Expiration time:', response.expires)
# After 1 second, the cached value will expired
time.sleep(1.2)
assert response.is_expired
response = session.get('https://httpbin.org/get')
assert not response.from_cache
def url_get(url):
"""
Download an URL using a cache and return the response object
:param url:
:return:
"""
s = None
info = dict()
log.debug("GET URL {!s}".format(url))
if 'file://' in url:
s = requests.session()
s.mount('file://', FileAdapter())
else:
s = CachedSession(cache_name="pyff_cache",
backend=config.request_cache_backend,
expire_after=config.request_cache_time,
old_data_on_error=True)
headers = {'User-Agent': "pyFF/{}".format(__version__), 'Accept': '*/*'}
r = s.get(url,
headers=headers,
verify=False,
timeout=config.request_timeout)
if config.request_override_encoding is not None:
r.encoding = config.request_override_encoding
return r
def url_get(url):
"""
Download an URL using a cache and return the response object
:param url:
:return:
"""
s = None
info = dict()
if 'file://' in url:
s = requests.session()
s.mount('file://', FileAdapter())
else:
s = CachedSession(cache_name="pyff_cache",
backend=config.request_cache_backend,
expire_after=config.request_cache_time,
old_data_on_error=True)
headers = {'User-Agent': "pyFF/{}".format(__version__), 'Accept': '*/*'}
try:
r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout)
except IOError as ex:
s = requests.Session()
r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout)
if six.PY2:
r.encoding = "utf-8"
log.debug("url_get({}) returns {} chrs encoded as {}".format(url, len(r.content), r.encoding))
if config.request_override_encoding is not None:
r.encoding = config.request_override_encoding
return r
def get_page_url():
"""
Initial function to get the list of page urls with respect to different alphabet.
:return: list of urls
"""
page_url_list = []
url = "http://devri.bzh/dictionnaire/a/"
session = CachedSession()
page = session.get(url)
soup = BeautifulSoup(page.content, "html.parser")
letter_list = soup.find_all("a", class_="enfant")
for i in tqdm(letter_list):
url_letter = "http://devri.bzh" + i["href"]
page_letter = session.get(url_letter)
soup_letter = BeautifulSoup(page_letter.content, "html.parser")
page_num = soup_letter.find("li", class_=["MarkupPagerNavLast MarkupPagerNavLastNum", "MarkupPagerNavLastNum"])
try:
page_num = page_num.a.text
url_list = [url_letter]
for j in range(2, int(page_num) + 1):
url_list.append(url_letter + f"page{j}")
page_url_list += url_list
except AttributeError:
pass
# add page for alphabet z manually
page_url_list += ["http://devri.bzh/dictionnaire/z/"]
return page_url_list
def session_obj(self):
session = CachedSession(allowable_methods=('GET', 'POST'),
ignored_parameters=['smpbss'])
if not IceDaily.RECAPTCHA:
with session.cache_disabled():
response = session.get(
url=
'https://www.theice.com/marketdata/reports/datawarehouse/ConsolidatedEndOfDayReportPDF.shtml',
headers={
'User-Agent': 'Mozilla/5.0',
'X-Requested-With': 'XMLHttpRequest'
},
params={
'selectionForm': '',
'exchangeCode': 'IFEU',
'optionRequest': self.flavor['optionRequest']
})
bs = BeautifulSoup(response.text)
df = pd.DataFrame([(opt['value'], opt.text)
for opt in bs.find_all('option')],
columns=["WebActiveCode", "ActiveName"])
df['ActiveCode'] = df.WebActiveCode.apply(
lambda s: s.split('|', 1)[1] if '|' in s else None)
df = df.dropna(how='any')
self.update_actives(df)
return session
def get_services(**options):
"""
This function builds the services dictionary, which is a simple dict of names-to-implementation used by bonobo
for runtime injection.
It will be used on top of the defaults provided by bonobo (fs, http, ...). You can override those defaults, or just
let the framework define them. You can also define your own services and naming is up to you.
:return: dict
"""
if options['use_cache']:
from requests_cache import CachedSession
servicenow = CachedSession('http.cache')
else:
servicenow = requests.Session()
servicenow.headers = {'User-Agent': 'Mozilla/ETL/v1'}
servicenow.auth = HTTPBasicAuth(options['sn_username'],
options['sn_password'])
servicenow.headers.update({'Accept-encoding': 'text/json'})
return {
'servicenow':
servicenow,
'db':
create_engine('sqlite:///test.sqlite', echo=False),
'vertica':
create_engine(options['vertica'].format(
username=options['vertica_username'],
password=options['vertica_password']),
echo=False)
}
def imdb_search(query):
session = CachedSession(expire_after=60 * 60 * 24)
stripped_query = "".join([x for x in query
if x.isalnum() or x == " "]).lower()
r = session.get("https://www.imdb.com/find",
params={
"q": stripped_query,
"s": "tt",
"ttype": "ft"
})
page_html = r.text
soup = BeautifulSoup(page_html, "html.parser")
results = []
for result in soup.select(".result_text"):
title = result.a.get_text()
imdb_id = result.a.get("href").split("/")
try:
imdb_id = result.a.get("href").split("/title/")[1].split("/")[0]
except IndexError:
imdb_id = None
continue
result.a.decompose()
try:
year = result.get_text().split("(")[1].split(")")[0]
int(year)
except:
year = None
continue
results.append((title, year, imdb_id))
return results
def test_urls_expire_after__evaluation_order(url, expected_expire_after):
"""If there are multiple matches, the first match should be used in the order defined"""
session = CachedSession(urls_expire_after={
'*.site_1.com/resource': 60 * 60 * 2,
'*.site_1.com': 60 * 60,
'*': 1,
}, )
assert session._url_expire_after(url) == expected_expire_after
def test_urls_expire_after(url, expected_expire_after):
session = CachedSession(urls_expire_after={
'*.site_1.com': 60 * 60,
'site_2.com/resource_1': 60 * 60 * 2,
'site_2.com/resource_2': 60 * 60 * 24,
'site_2.com/static': -1,
}, )
assert session._url_expire_after(url) == expected_expire_after
def wrapper(page_html=None, imdb_id=None, soup=None, *args, **kwargs):
if not (page_html or imdb_id or soup):
return None
if imdb_id:
session = CachedSession(expire_after=60 * 60 * 24)
r = session.get("https://www.imdb.com/title/{}".format(imdb_id))
page_html = r.text
soup = soup or BeautifulSoup(page_html, "html.parser")
return f(soup, *args, **kwargs)
def test_post_parameters_normalization(self):
params = {"a": "a", "b": ["1", "2", "3"], "c": "4"}
url = httpbin("post")
s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('GET', 'POST'))
self.assertFalse(s.post(url, data=params).from_cache)
self.assertTrue(s.post(url, data=params).from_cache)
self.assertTrue(s.post(url, data=sorted(params.items())).from_cache)
self.assertFalse(s.post(url, data=sorted(params.items(), reverse=True)).from_cache)
def __init__(self):
super(HelioViewerSkill, self).__init__(name="HelioViewerSkill")
self.session = CachedSession(backend='memory',
expire_after=timedelta(hours=6))
self.translate_cache = {} # save calls to avoid ip banning
self.img_cache = {} # dont re-parse for speed
self.current_date = datetime.now()
self.current_camera = "sunspots"
create_daemon(self.bootstrap)
def test_throttle_cache(self):
url = httpbin('get')
s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=0.06)
s.throttle(url, 0.5) # one every 2 seconds
r = s.get(url)
time.sleep(0.6)
t = time.time()
r = s.get(url)
delta = time.time() - t
self.assertGreaterEqual(delta, 1)
def get_data(leaderboard, year):
lb_path = LEADERBOARDS.get(leaderboard, leaderboard)
url = f"https://adventofcode.com/{year}/leaderboard/private/view/{lb_path}"
cache_options = {"backend": "filesystem", "expire_after": 86400}
session = CachedSession("../../site_cache", **cache_options)
cookies = dict([Path("../.session-cookie").read_text().strip().split("=")])
response = session.get(url, cookies=cookies)
if response.status_code != 200:
pdb.set_trace()
return response.json()
def get_session() -> CachedSession:
"""Make a cached session."""
path = settings.STATE_PATH.joinpath("http").as_posix()
session = CachedSession(cache_name=path,
expire_after=settings.CACHE_EXPIRE)
session.headers.update(HEADERS)
# weird monkey-patch: default timeout for requests sessions
session.request = functools.partial(session.request,
timeout=settings.HTTP_TIMEOUT)
return session
def get_page_content(url: str) -> str:
"""
Utilization function to get page content literally
:param url: string
:return: string
"""
session = CachedSession()
page = session.get(url)
soup = BeautifulSoup(page.content, "html.parser")
content = soup.prettify()
return content
def fetch_df(url):
r = requests.get(url)
js = r.json()
if "features" not in js:
s = CachedSession()
with s.cache_disabled():
r = s.get(url)
js = r.json()
features = [feature["attributes"] for feature in js["features"]]
df = pd.DataFrame(features)
return df
def get(self, resource: str, params=None):
client = BackendApplicationClient(client_id=self.client_id)
oauth = OAuth2Session(client=client)
token = oauth.fetch_token(token_url='https://us.battle.net/oauth/token', client_id=self.client_id, client_secret=self.client_secret)
if params is None:
params = {}
root = 'https://us.api.blizzard.com/'
params.update({'locale': 'en_US', 'namespace': 'static-us', 'access_token': token['access_token']})
session = CachedSession(expire_after=timedelta(hours=1))
return session.get(root + resource, params=params).json()
def __init__(self, url, cache=False, cachetime=300):
super(PlexServer, self).__init__(url)
session = None # use the default session
cachetime = cachetime or None
if CachedSession and cache and isinstance(cache, six.string_types):
if cache == 'memory':
session = CachedSession(cache, cache, cachetime)
elif cache == 'sqlite':
session = CachedSession('request-cache',
'sqlite',
cachetime,
fast_save=True)
self._config = RequestConfig(session=session)
def wrapper(title, year, *args, **kwargs):
session = CachedSession(expire_after=60 * 60 * 24)
# https://www.rottentomatoes.com/napi/search/?query=parasite&offset=0&limit=10
stripped_title = "".join([x for x in title
if x.isalnum() or x == " "]).lower().strip()
response = session.get("https://www.rottentomatoes.com/napi/search",
params={
"query": stripped_title,
"offset": 0,
"limit": 10
})
json = response.json()
return f(stripped_title, year, json, *args, **kwargs)
def __init__(self,
registry_config="app-registry.yaml",
product="common",
stub=False):
self.registry = self._get_registry(registry_config, product=product)
""" Uncomment this and related lines when this code goes live,.
Use a timeout on the API so the unit tests are not slowed down. """
if not os.environ.get('DEV_PHASE') == 'stub':
self.client = TychoClient(
url=os.environ.get('TYCHO_URL', "http://localhost:5000"))
self.product = product
self.apps = self._grok()
self.http_session = CachedSession(cache_name='tycho-registry')
def test_expire_cache(self):
delay = 1
url = httpbin('delay/%s' % delay)
s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=0.06)
t = time.time()
r = s.get(url)
delta = time.time() - t
self.assertGreaterEqual(delta, delay)
time.sleep(0.5)
t = time.time()
r = s.get(url)
delta = time.time() - t
self.assertGreaterEqual(delta, delay)
def __init__(self, data_dir: Optional[str] = None):
super().__init__(data_dir)
env_var = f'{__name__:s}.requests_cache'.replace('.', '_').upper()
fn = os.environ.get(env_var, None)
if fn is None:
logging.debug(f'Environment variable {env_var} not found.')
fn = os.path.join(self.get_data_dir(), 'requests_cache.sqlite')
logging.warning(f'Using requests cache at {os.path.abspath(fn)}')
fn = fn[:-len('.sqlite')] if fn.endswith('.sqlite') else fn
self.__cached_session = CachedSession(cache_name=fn)
self.__hash_constructors: Dict[str, callable] = dict()
def test_expire_cache(self):
delay = 1
url = httpbin('delay/%s' % delay)
s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=0.06)
t = time.time()
r = s.get(url)
delta = time.time() - t
self.assertGreaterEqual(delta, delay)
time.sleep(0.5)
t = time.time()
r = s.get(url)
delta = time.time() - t
self.assertGreaterEqual(delta, delay)
def __init__(
self,
allowed_licenses=None,
excluded_packages=None,
includes=None,
cache_name='~/.cache/pre-commit-license-check.sqlite') -> None:
self.allowed_licenses = allowed_licenses if allowed_licenses else []
self.excluded_packages = excluded_packages if excluded_packages else []
self.includes = includes if includes else []
self.session = CachedSession(cache_name,
expire_after=timedelta(days=1))
def test_ignore_parameters_post_raw(self):
url = httpbin("post")
ignored_param = "ignored"
raw_data = "raw test data"
s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('POST'),
ignored_parameters=[ignored_param])
self.assertFalse(s.post(url, data=raw_data).from_cache)
self.assertTrue(s.post(url, data=raw_data).from_cache)
raw_data = "new raw data"
self.assertFalse(s.post(url, data=raw_data).from_cache)
def test_post_params(self):
# issue #2
self.s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('GET', 'POST'))
d = {'param1': 'test1'}
for _ in range(2):
self.assertEqual(self.post(d)['form'], d)
d = {'param1': 'test1', 'param3': 'test3'}
self.assertEqual(self.post(d)['form'], d)
self.assertTrue(self.s.post(httpbin('post'), data=d).from_cache)
d.update({'something': 'else'})
self.assertFalse(self.s.post(httpbin('post'), data=d).from_cache)
def __init__(self):
super(HubbleTelescopeSkill, self).__init__(name="HubbleTelescopeSkill")
if "random" not in self.settings:
# idle screen, random or latest
self.settings["random"] = True
if "include_james_webb" not in self.settings:
self.settings["include_james_webb"] = False
if "exclude_long" not in self.settings:
self.settings["exclude_long"] = True
self.session = CachedSession(backend='memory',
expire_after=timedelta(hours=6))
self.translate_cache = {}
# bootstrap - cache image data
create_daemon(self.latest_hubble)
def get_profile(request, provider_name):
provider = SOCIAL_AUTH_PROVIDERS[provider_name]()
# Prepare the session for fetching the token.
session = OAuth2Session(
client_id=provider.client_id,
scope=provider.scope,
state=request.session.get('social_login_state', ''),
redirect_uri=request.build_absolute_uri(
reverse('social_login_callback',
kwargs={'provider_name': provider_name})),
)
# Clear the session state data before continuing.
if 'social_login_state' in request.session:
del request.session['social_login_state']
# Go and fetch the oauth token.
token = session.fetch_token(
token_url=provider.token_uri,
client_secret=provider.client_secret,
authorization_response=request.build_absolute_uri())
# Get the id_token from the oauth token.
unparsed_id_token = token['id_token']
# Retrieve the certificates from the provider, this is in json format and is cached for 1 hour.
expire_after = datetime.timedelta(hours=1)
cached_session = CachedSession(backend='memory', expire_after=expire_after)
provider_certificates = cached_session.get(
provider.jwks_uri).json().get('keys')
# Put the certificates in a dict, with the identifier as key.
certificate_set = {cert['kid']: cert for cert in provider_certificates}
# Look up which certificate was used to sign the id_token.
kid = jwt.get_unverified_header(unparsed_id_token).get('kid')
# Convert the certificate from json to something the jwt library can use.
certificate = RSAAlgorithm.from_jwk(json.dumps(certificate_set[kid]))
# Now finally do the actual decoding and verifying.
id_token = jwt.decode(unparsed_id_token,
certificate,
audience=provider.client_id)
# Put the parsed id_token in the response.
token['id_token'] = id_token
return provider.parse_profile(session, token)
def test_ignore_parameters_post_raw(self):
url = httpbin("post")
ignored_param = "ignored"
raw_data = "raw test data"
s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('POST'),
ignored_parameters=[ignored_param])
self.assertFalse(s.post(url, data=raw_data).from_cache)
self.assertTrue(s.post(url, data=raw_data).from_cache)
raw_data = "new raw data"
self.assertFalse(s.post(url, data=raw_data).from_cache)
def api_get(endpoint, query, cache=False):
if not cache:
s = CachedSession()
with s.cache_disabled():
r = s.get(
app.config['API_URL'] + endpoint + '?q=' + urllib.parse.quote_plus(json.dumps(query)),
headers=gen_api_header(),
verify=app.config['VERIFY_SSL'])
else:
r = requests.get(
app.config['API_URL'] + endpoint + '?q=' + urllib.parse.quote_plus(json.dumps(query)),
headers=gen_api_header(),
verify=app.config['VERIFY_SSL'])
if r.status_code == 200:
# If created then it returns the object data
return json.loads(r.text).get('objects')
else:
return {}
def test_ignore_parameters_get(self):
url = httpbin("get")
ignored_param = "ignored"
usual_param = "some"
params = {ignored_param: "1", usual_param: "1"}
s = CachedSession(CACHE_NAME, CACHE_BACKEND,
ignored_parameters=[ignored_param])
r = s.get(url, params=params)
self.assertIn(ignored_param, r.json()['args'].keys())
self.assertFalse(r.from_cache)
self.assertTrue(s.get(url, params=params).from_cache)
params[ignored_param] = "new"
self.assertTrue(s.get(url, params=params).from_cache)
params[usual_param] = "new"
self.assertFalse(s.get(url, params=params).from_cache)
def test_headers_in_get_query(self):
url = httpbin("get")
s = CachedSession(CACHE_NAME, CACHE_BACKEND, include_get_headers=True)
headers = {"Accept": "text/json"}
self.assertFalse(s.get(url, headers=headers).from_cache)
self.assertTrue(s.get(url, headers=headers).from_cache)
headers["Accept"] = "text/xml"
self.assertFalse(s.get(url, headers=headers).from_cache)
self.assertTrue(s.get(url, headers=headers).from_cache)
headers["X-custom-header"] = "custom"
self.assertFalse(s.get(url, headers=headers).from_cache)
self.assertTrue(s.get(url, headers=headers).from_cache)
self.assertFalse(s.get(url).from_cache)
self.assertTrue(s.get(url).from_cache)
def test_ignore_parameters_post_json(self):
url = httpbin("post")
ignored_param = "ignored"
usual_param = "some"
d = {ignored_param: "1", usual_param: "1"}
s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('POST'),
ignored_parameters=[ignored_param])
r = s.post(url, json=d)
self.assertIn(ignored_param, json.loads(r.json()['data']).keys())
self.assertFalse(r.from_cache)
self.assertTrue(s.post(url, json=d).from_cache)
d[ignored_param] = "new"
self.assertTrue(s.post(url, json=d).from_cache)
d[usual_param] = "new"
self.assertFalse(s.post(url, json=d).from_cache)
def test_post_params(self):
# issue #2
self.s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('GET', 'POST'))
d = {'param1': 'test1'}
for _ in range(2):
self.assertEqual(self.post(d)['form'], d)
d = {'param1': 'test1', 'param3': 'test3'}
self.assertEqual(self.post(d)['form'], d)
self.assertTrue(self.s.post(httpbin('post'), data=d).from_cache)
d.update({'something': 'else'})
self.assertFalse(self.s.post(httpbin('post'), data=d).from_cache)
def test_remove_expired_entries(self, datetime_mock, datetime_mock2):
expire_after = timedelta(minutes=10)
start_time = datetime.utcnow().replace(year=2010, minute=0)
datetime_mock.utcnow.return_value = start_time
datetime_mock2.utcnow.return_value = start_time
s = CachedSession(CACHE_NAME, CACHE_BACKEND, expire_after=expire_after)
s.get(httpbin('get'))
s.get(httpbin('relative-redirect/3'))
datetime_mock.utcnow.return_value = start_time + expire_after * 2
datetime_mock2.utcnow.return_value = datetime_mock.utcnow.return_value
ok_url = 'get?x=1'
s.get(httpbin(ok_url))
self.assertEqual(len(s.cache.responses), 3)
self.assertEqual(len(s.cache.keys_map), 3)
s.remove_expired_responses()
self.assertEqual(len(s.cache.responses), 1)
self.assertEqual(len(s.cache.keys_map), 0)
self.assertIn(ok_url, list(s.cache.responses.values())[0][0].url)
def __init__(self,
url=None,
max_file_size=None,
allowed_content_types=(),
jsfetcher=None,
marker=r'<meta.*name="Generator".*>'):
logger.info('Running downloader for url: %s' % url)
self.url = str(url)
self.max_file_size = max_file_size or settings.RFDOCS.get('MAX_FILE_SIZE')
self.allowed_content_types = allowed_content_types or settings.RFDOCS.get('ALLOWED_CONTENT_TYPES')
self.s = CachedSession()
self.r = None
self.error = {}
self.jsfetcher = jsfetcher
self.marker = marker
def test_ignore_cgi_parameter(self):
s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=5)
s.ignore_cgi('foo')
url = httpbin('get?foo=1')
r = s.get(url)
self.assertFalse(r.from_cache)
r = s.get(url)
self.assertTrue(r.from_cache)
url = httpbin('get')
r = s.get(url)
self.assertTrue(r.from_cache)
def test_ignore_parameters(self):
url = httpbin("get")
ignored_param = "ignored"
usual_param = "some"
s = CachedSession(CACHE_NAME, CACHE_BACKEND, ignored_parameters=[ignored_param])
params = {ignored_param: "1", usual_param: "1"}
s.get(url, params=params)
self.assertTrue(s.get(url, params=params).from_cache)
params[ignored_param] = "new"
self.assertTrue(s.get(url, params=params).from_cache)
params[usual_param] = "new"
self.assertFalse(s.get(url, params=params).from_cache)
def test_post_data(self):
# issue #2, raw payload
self.s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('GET', 'POST'))
d1 = json.dumps({'param1': 'test1'})
d2 = json.dumps({'param1': 'test1', 'param2': 'test2'})
d3 = str('some unicode data')
if is_py3:
bin_data = bytes('some binary data', 'utf8')
else:
bin_data = bytes('some binary data')
for d in (d1, d2, d3):
self.assertEqual(self.post(d)['data'], d)
r = self.s.post(httpbin('post'), data=d)
self.assert_(hasattr(r, 'from_cache'))
self.assertEqual(self.post(bin_data)['data'],
bin_data.decode('utf8'))
r = self.s.post(httpbin('post'), data=bin_data)
self.assert_(hasattr(r, 'from_cache'))
class CacheTestCase(unittest.TestCase):
def setUp(self):
self.s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE)
self.s.cache.clear()
requests_cache.uninstall_cache()
def test_expire_cache(self):
delay = 1
url = httpbin('delay/%s' % delay)
s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=0.06)
t = time.time()
r = s.get(url)
delta = time.time() - t
self.assertGreaterEqual(delta, delay)
time.sleep(0.5)
t = time.time()
r = s.get(url)
delta = time.time() - t
self.assertGreaterEqual(delta, delay)
def test_delete_urls(self):
url = httpbin('relative-redirect/3')
r = self.s.get(url)
for i in range(1, 4):
self.assert_(self.s.cache.has_url(httpbin('relative-redirect/%s' % i)))
self.s.cache.delete_url(url)
self.assert_(not self.s.cache.has_url(url))
def test_unregistered_backend(self):
with self.assertRaises(ValueError):
CachedSession(CACHE_NAME, backend='nonexistent')
def test_hooks(self):
state = defaultdict(int)
for hook in ('response',): # TODO it's only one hook here
def hook_func(r, *args, **kwargs):
state[hook] += 1
return r
n = 5
for i in range(n):
r = self.s.get(httpbin('get'), hooks={hook: hook_func})
self.assertEqual(state[hook], n)
def test_attr_from_cache_in_hook(self):
state = defaultdict(int)
hook = 'response'
def hook_func(r, *args, **kwargs):
if state[hook] > 0:
self.assert_(r.from_cache, True)
state[hook] += 1
return r
n = 5
for i in range(n):
r = self.s.get(httpbin('get'), hooks={hook: hook_func})
self.assertEqual(state[hook], n)
def test_post(self):
url = httpbin('post')
r1 = json.loads(self.s.post(url, data={'test1': 'test1'}).text)
r2 = json.loads(self.s.post(url, data={'test2': 'test2'}).text)
self.assertIn('test2', r2['form'])
req = Request('POST', url).prepare()
self.assert_(not self.s.cache.has_key(self.s.cache.create_key(req)))
def test_disabled(self):
url = httpbin('get')
requests_cache.install_cache(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE)
requests.get(url)
with requests_cache.disabled():
for i in range(2):
r = requests.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
with self.s.cache_disabled():
for i in range(2):
r = self.s.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
r = self.s.get(url)
self.assertTrue(getattr(r, 'from_cache', False))
def test_enabled(self):
url = httpbin('get')
options = dict(cache_name=CACHE_NAME, backend=CACHE_BACKEND,
fast_save=FAST_SAVE)
with requests_cache.enabled(**options):
r = requests.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
for i in range(2):
r = requests.get(url)
self.assertTrue(getattr(r, 'from_cache', False))
r = requests.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
def test_content_and_cookies(self):
requests_cache.install_cache(CACHE_NAME, CACHE_BACKEND)
s = requests.session()
def js(url):
return json.loads(s.get(url).text)
r1 = js(httpbin('cookies/set/test1/test2'))
with requests_cache.disabled():
r2 = js(httpbin('cookies'))
self.assertEqual(r1, r2)
r3 = js(httpbin('cookies'))
with requests_cache.disabled():
r4 = js(httpbin('cookies/set/test3/test4'))
# from cache
self.assertEqual(r3, js(httpbin('cookies')))
# updated
with requests_cache.disabled():
self.assertEqual(r4, js(httpbin('cookies')))
def test_response_history(self):
r1 = self.s.get(httpbin('relative-redirect/3'))
def test_redirect_history(url):
r2 = self.s.get(url)
self.assertTrue(r2.from_cache)
for r11, r22 in zip(r1.history, r2.history):
self.assertEqual(r11.url, r22.url)
test_redirect_history(httpbin('relative-redirect/3'))
test_redirect_history(httpbin('relative-redirect/2'))
r3 = requests.get(httpbin('relative-redirect/1'))
self.assertEqual(len(r3.history), 1)
def test_response_history_simple(self):
r1 = self.s.get(httpbin('relative-redirect/2'))
r2 = self.s.get(httpbin('relative-redirect/1'))
self.assertTrue(r2.from_cache)
def post(self, data):
return json.loads(self.s.post(httpbin('post'), data=data).text)
def test_post_params(self):
# issue #2
self.s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('GET', 'POST'))
d = {'param1': 'test1'}
for _ in range(2):
self.assertEqual(self.post(d)['form'], d)
d = {'param1': 'test1', 'param3': 'test3'}
self.assertEqual(self.post(d)['form'], d)
self.assertTrue(self.s.post(httpbin('post'), data=d).from_cache)
d.update({'something': 'else'})
self.assertFalse(self.s.post(httpbin('post'), data=d).from_cache)
def test_post_data(self):
# issue #2, raw payload
self.s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('GET', 'POST'))
d1 = json.dumps({'param1': 'test1'})
d2 = json.dumps({'param1': 'test1', 'param2': 'test2'})
d3 = str('some unicode data')
if is_py3:
bin_data = bytes('some binary data', 'utf8')
else:
bin_data = bytes('some binary data')
for d in (d1, d2, d3):
self.assertEqual(self.post(d)['data'], d)
r = self.s.post(httpbin('post'), data=d)
self.assert_(hasattr(r, 'from_cache'))
self.assertEqual(self.post(bin_data)['data'],
bin_data.decode('utf8'))
r = self.s.post(httpbin('post'), data=bin_data)
self.assert_(hasattr(r, 'from_cache'))
def test_get_params_as_argument(self):
for _ in range(5):
p = {'arg1': 'value1'}
r = self.s.get(httpbin('get'), params=p)
self.assert_(self.s.cache.has_url( httpbin('get?arg1=value1')))
def test_https_support(self):
n = 10
delay = 1
url = 'https://httpbin.org/delay/%s?ar1=value1' % delay
t = time.time()
for _ in range(n):
r = self.s.get(url, verify=False)
self.assertLessEqual(time.time() - t, delay * n / 2)
def test_from_cache_attribute(self):
url = httpbin('get?q=1')
self.assertFalse(self.s.get(url).from_cache)
self.assertTrue(self.s.get(url).from_cache)
self.s.cache.clear()
self.assertFalse(self.s.get(url).from_cache)
def test_gzip_response(self):
url = httpbin('gzip')
self.assertFalse(self.s.get(url).from_cache)
self.assertTrue(self.s.get(url).from_cache)
def test_close_response(self):
for _ in range(3):
r = self.s.get(httpbin("get"))
r.close()
def test_get_parameters_normalization(self):
url = httpbin("get")
params = {"a": "a", "b": ["1", "2", "3"], "c": "4"}
self.assertFalse(self.s.get(url, params=params).from_cache)
r = self.s.get(url, params=params)
self.assertTrue(r.from_cache)
self.assertEquals(r.json()["args"], params)
self.assertFalse(self.s.get(url, params={"a": "b"}).from_cache)
self.assertTrue(self.s.get(url, params=sorted(params.items())).from_cache)
class UserSubclass(dict):
def items(self):
return sorted(super(UserSubclass, self).items(), reverse=True)
params["z"] = "5"
custom_dict = UserSubclass(params)
self.assertFalse(self.s.get(url, params=custom_dict).from_cache)
self.assertTrue(self.s.get(url, params=custom_dict).from_cache)
def test_post_parameters_normalization(self):
params = {"a": "a", "b": ["1", "2", "3"], "c": "4"}
url = httpbin("post")
s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('GET', 'POST'))
self.assertFalse(s.post(url, data=params).from_cache)
self.assertTrue(s.post(url, data=params).from_cache)
self.assertTrue(s.post(url, data=sorted(params.items())).from_cache)
self.assertFalse(s.post(url, data=sorted(params.items(), reverse=True)).from_cache)
def test_stream_requests_support(self):
n = 100
url = httpbin("stream/%s" % n)
r = self.s.get(url, stream=True)
lines = list(r.iter_lines())
self.assertEquals(len(lines), n)
for i in range(2):
r = self.s.get(url, stream=True)
self.assertTrue(r.from_cache)
cached_lines = list(r.iter_lines())
self.assertEquals(cached_lines, lines)
def test_headers_in_get_query(self):
url = httpbin("get")
s = CachedSession(CACHE_NAME, CACHE_BACKEND, include_get_headers=True)
headers = {"Accept": "text/json"}
self.assertFalse(s.get(url, headers=headers).from_cache)
self.assertTrue(s.get(url, headers=headers).from_cache)
headers["Accept"] = "text/xml"
self.assertFalse(s.get(url, headers=headers).from_cache)
self.assertTrue(s.get(url, headers=headers).from_cache)
headers["X-custom-header"] = "custom"
self.assertFalse(s.get(url, headers=headers).from_cache)
self.assertTrue(s.get(url, headers=headers).from_cache)
self.assertFalse(s.get(url).from_cache)
self.assertTrue(s.get(url).from_cache)
def test_str_and_repr(self):
s = repr(CachedSession(CACHE_NAME, CACHE_BACKEND, expire_after=10))
self.assertIn(CACHE_NAME, s)
self.assertIn("10", s)
@mock.patch("requests_cache.core.datetime")
def test_return_old_data_on_error(self, datetime_mock):
datetime_mock.utcnow.return_value = datetime.utcnow()
expire_after = 100
url = httpbin("get")
s = CachedSession(CACHE_NAME, CACHE_BACKEND, old_data_on_error=True, expire_after=expire_after)
header = "X-Tst"
def get(n):
return s.get(url, headers={header: n}).json()["headers"][header]
get("expired")
self.assertEquals(get("2"), "expired")
datetime_mock.utcnow.return_value = datetime.utcnow() + timedelta(seconds=expire_after * 2)
with mock.patch.object(s.cache, "save_response", side_effect=Exception):
self.assertEquals(get("3"), "expired")
with mock.patch("requests_cache.core.OriginalSession.send") as send_mock:
resp_mock = requests.Response()
request = requests.Request("GET", url)
resp_mock.request = request.prepare()
resp_mock.status_code = 400
resp_mock._content = '{"other": "content"}'
send_mock.return_value = resp_mock
self.assertEquals(get("3"), "expired")
resp_mock.status_code = 200
self.assertIs(s.get(url).content, resp_mock.content)
# default behaviour
datetime_mock.return_value = datetime.utcnow() + timedelta(seconds=expire_after * 2)
s = CachedSession(CACHE_NAME, CACHE_BACKEND, old_data_on_error=False, expire_after=100)
with mock.patch.object(s.cache, "save_response", side_effect=Exception):
with self.assertRaises(Exception):
s.get(url)
def test_ignore_parameters_get(self):
url = httpbin("get")
ignored_param = "ignored"
usual_param = "some"
params = {ignored_param: "1", usual_param: "1"}
s = CachedSession(CACHE_NAME, CACHE_BACKEND,
ignored_parameters=[ignored_param])
r = s.get(url, params=params)
self.assertIn(ignored_param, r.json()['args'].keys())
self.assertFalse(r.from_cache)
self.assertTrue(s.get(url, params=params).from_cache)
params[ignored_param] = "new"
self.assertTrue(s.get(url, params=params).from_cache)
params[usual_param] = "new"
self.assertFalse(s.get(url, params=params).from_cache)
def test_ignore_parameters_post(self):
url = httpbin("post")
ignored_param = "ignored"
usual_param = "some"
d = {ignored_param: "1", usual_param: "1"}
s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('POST'),
ignored_parameters=[ignored_param])
r = s.post(url, data=d)
self.assertIn(ignored_param, r.json()['form'].keys())
self.assertFalse(r.from_cache)
self.assertTrue(s.post(url, data=d).from_cache)
d[ignored_param] = "new"
self.assertTrue(s.post(url, data=d).from_cache)
d[usual_param] = "new"
self.assertFalse(s.post(url, data=d).from_cache)
def test_ignore_parameters_post_json(self):
url = httpbin("post")
ignored_param = "ignored"
usual_param = "some"
d = {ignored_param: "1", usual_param: "1"}
s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('POST'),
ignored_parameters=[ignored_param])
r = s.post(url, json=d)
self.assertIn(ignored_param, json.loads(r.json()['data']).keys())
self.assertFalse(r.from_cache)
self.assertTrue(s.post(url, json=d).from_cache)
d[ignored_param] = "new"
self.assertTrue(s.post(url, json=d).from_cache)
d[usual_param] = "new"
self.assertFalse(s.post(url, json=d).from_cache)
def test_ignore_parameters_post_raw(self):
url = httpbin("post")
ignored_param = "ignored"
raw_data = "raw test data"
s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('POST'),
ignored_parameters=[ignored_param])
self.assertFalse(s.post(url, data=raw_data).from_cache)
self.assertTrue(s.post(url, data=raw_data).from_cache)
raw_data = "new raw data"
self.assertFalse(s.post(url, data=raw_data).from_cache)
@mock.patch("requests_cache.backends.base.datetime")
@mock.patch("requests_cache.core.datetime")
def test_remove_expired_entries(self, datetime_mock, datetime_mock2):
expire_after = timedelta(minutes=10)
start_time = datetime.utcnow().replace(year=2010, minute=0)
datetime_mock.utcnow.return_value = start_time
datetime_mock2.utcnow.return_value = start_time
s = CachedSession(CACHE_NAME, CACHE_BACKEND, expire_after=expire_after)
s.get(httpbin('get'))
s.get(httpbin('relative-redirect/3'))
datetime_mock.utcnow.return_value = start_time + expire_after * 2
datetime_mock2.utcnow.return_value = datetime_mock.utcnow.return_value
ok_url = 'get?x=1'
s.get(httpbin(ok_url))
self.assertEqual(len(s.cache.responses), 3)
self.assertEqual(len(s.cache.keys_map), 3)
s.remove_expired_responses()
self.assertEqual(len(s.cache.responses), 1)
self.assertEqual(len(s.cache.keys_map), 0)
self.assertIn(ok_url, list(s.cache.responses.values())[0][0].url)
class DownloadExternalResource(object):
def __init__(self,
url=None,
max_file_size=None,
allowed_content_types=(),
jsfetcher=None,
marker=r'<meta.*name="Generator".*>'):
logger.info('Running downloader for url: %s' % url)
self.url = str(url)
self.max_file_size = max_file_size or settings.RFDOCS.get('MAX_FILE_SIZE')
self.allowed_content_types = allowed_content_types or settings.RFDOCS.get('ALLOWED_CONTENT_TYPES')
self.s = CachedSession()
self.r = None
self.error = {}
self.jsfetcher = jsfetcher
self.marker = marker
def _set_error(self, key, value):
self.error[key] = value
return self.error
def _no_error(self):
self.error = {}
return self.error
def send_request(self):
logger.info('Download external resource: %s' % self.url)
try:
self.r = self.s.get(self.url)
except (requests.exceptions.MissingSchema,
requests.exceptions.InvalidSchema,
requests.exceptions.ConnectionError,) as error:
logger.warn("Failed to download resource. Error: %s" % error)
self._set_error('error', error)
def get_response(self):
# Workaround for Robot Framework libraries with version >= 2.8 (or even 2.7).
# Libraries are generated with JQuery templates system.
# Python's `requests` module fetches raw content, of course not rendered HTML.
# Thus parser will fail in such cases.
# For this reason an awesome PhantomJS is used. Unfortunately `requests_cache` module
# wont help here because PhantomJS has to request the URL by its
# own methods to be able to render HTML.
# One more notice is that I don't use PhantomJS native features to actually write data to filesystem.
# (PhantomJS has `fs` module to work with filesystem).
# Why? I don't know yet :)
# Instead use `subprocess.Popen` to execute javascript code,
# let PhantomJS to do his job and give the output back to python.
# Anyway I want the resource to be validated by Django's validators before we can proceed.
# So if we came here, the validation has passed.
# First, check if response content has
# meta tag with name="Generator"
# If so, the document uses JQuery templates plugin system and we need help of PhantomJS.
content = self.r.content
mo = re.search(self.marker, content, re.DOTALL | re.M | re.I)
if not mo:
return content
logger.info('Using phantomjs to download resource')
alternate_downloader = PhantomJSHelper(url=self.url, error_callback=self._set_error)
return alternate_downloader.get_content()
def validate_response(self):
logger.info('Validate external resource: %s' % self.url)
if not self.r:
self.send_request()
if self.error:
return self.error
try:
self.r.raise_for_status()
except requests.exceptions.HTTPError as error:
logger.warn("Failed to fetch resource. Error: %s" % error)
return self._set_error('error', error)
if self.r.status_code == requests.codes.ok:
content_len = self.r.headers.get('content-length', None)
if content_len:
csize = int(content_len)
else:
csize = len(self.r.content)
if csize < self.max_file_size:
ctype_header = self.r.headers.get('content-type')
if not ctype_header:
return self._set_error('content_type',
'Response does not contain the \'Content-Type\' header. Rejected.')
ctype = ctype_header.split(';')[0].lower()
if ctype in [ct.lower() for ct in self.allowed_content_types]:
# the place where all the procedure passed and we succeeded
return self._no_error()
else:
logger.warn("Failed to fetch resource. "
"Allowed content types are: %s."
"The content type is: %s" % (', '.join(self.allowed_content_types), ctype,))
return self._set_error('content_type', ctype)
else:
logger.warn("Failed to fetch resource. "
"The content size \'%s\' exceeds maximum allowed size: %s" % (self.max_file_size, csize))
return self._set_error('content_size', csize)
else:
error = requests.exceptions.HTTPError(self.r.status_code)
logger.warn("Failed to fetch resource. Error: %s" % error)
return self._set_error('error', error)
def get_response_from_cache(self):
if not self.s.cache.has_url(self.url):
self.send_request()
self.validate_response()
return self.get_response()
self.r = self.s.get(self.url)
return self.get_response()
def get_response_from_cache_or_raise_error(self):
response = self.get_response_from_cache()
if self.error:
err = self.error.get('error')
# these are model's `clean` ValidationError (not the same as forms.ValidationError)
if err:
raise exceptions.ValidationError(err)
else:
raise exceptions.ValidationError(self.error)
return response
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import requests
from requests_cache import CachedSession
cs = CachedSession(allowable_methods=('GET', 'POST'))
cs.cache.clear()
for i in range(2):
r = cs.get("http://httpbin.org/get?p1=v1", params={'p2': 'v2', 'p3': 'cyrЯЯ'})
print r
print r.from_cache
import sys
import time
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
HEADER = ['url', 'link', 'title', 'description', 'content', 'topics', 'organisations']
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('input_file', nargs='?', type=argparse.FileType('r'), default=sys.stdin)
parser.add_argument('--environment', '-e', dest='root_url', default='https://www-origin.staging.publishing.service.gov.uk', help='the environment used to query the search API')
parser.add_argument('--skip', '-s', dest='skip', type=int, default=0, help='Number of input rows to skip. Can be used to resume a partially completed import')
parser.add_argument('--skip-redirects', '-r', dest='skip_redirects', action='store_true', help="Don't test URLs on GOV.UK to resolve redirected links.")
parser.add_argument('--wait-time', '-w', dest='wait_time', type=float, default=0.1, help='Time to wait between each link, to work around rate limiting.')
args = parser.parse_args()
session = CachedSession(cache_name='govuk_cache', backend='sqlite')
retries = Retry(total=5, backoff_factor=args.wait_time, status_forcelist=[ 429 ])
session.mount('http://', HTTPAdapter(max_retries=retries))
session.mount('https://', HTTPAdapter(max_retries=retries))
def test_base_path(original_base_path, args):
"""
Given a base path, try and classify it as valid, redirected, or gone,
so that we can fetch data even when the link has been redirected.
If it can't be retrieved, return None, otherwise return the ultimate base path.
We might include the same document multiple times in our analysis, but
this should only happen for a small amount of links and we can strip
out duplicates later.
class CacheTestCase(unittest.TestCase):
def setUp(self):
self.s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE)
self.s.cache.clear()
requests_cache.uninstall_cache()
def test_expire_cache(self):
delay = 1
url = httpbin('delay/%s' % delay)
s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=0.06)
t = time.time()
r = s.get(url)
delta = time.time() - t
self.assertGreaterEqual(delta, delay)
time.sleep(0.5)
t = time.time()
r = s.get(url)
delta = time.time() - t
self.assertGreaterEqual(delta, delay)
def test_delete_urls(self):
url = httpbin('redirect/3')
r = self.s.get(url)
for i in range(1, 4):
self.assert_(self.s.cache.has_url(httpbin('redirect/%s' % i)))
self.s.cache.delete_url(url)
self.assert_(not self.s.cache.has_url(url))
def test_unregistered_backend(self):
with self.assertRaises(ValueError):
CachedSession(CACHE_NAME, backend='nonexistent')
# def test_async_compatibility(self):
# try:
# import grequests
# except Exception:
# self.skipTest('gevent is not installed')
# n = 3
# def long_running():
# t = time.time()
# rs = [grequests.get(httpbin('delay/%s' % i)) for i in range(n + 1)]
# grequests.map(rs)
# return time.time() - t
# # cache it
# delta = long_running()
# self.assertGreaterEqual(delta, n)
# # fast from cache
# delta = 0
# for i in range(n):
# delta += long_running()
# self.assertLessEqual(delta, 1)
def test_hooks(self):
state = defaultdict(int)
for hook in ('response',): # TODO it's only one hook here
def hook_func(r, *args, **kwargs):
state[hook] += 1
return r
n = 5
for i in range(n):
r = self.s.get(httpbin('get'), hooks={hook: hook_func})
self.assertEqual(state[hook], n)
def test_attr_from_cache_in_hook(self):
state = defaultdict(int)
hook = 'response'
def hook_func(r, *args, **kwargs):
if state[hook] > 0:
self.assert_(r.from_cache, True)
state[hook] += 1
return r
n = 5
for i in range(n):
r = self.s.get(httpbin('get'), hooks={hook: hook_func})
self.assertEqual(state[hook], n)
def test_post(self):
url = httpbin('post')
r1 = json.loads(self.s.post(url, data={'test1': 'test1'}).text)
r2 = json.loads(self.s.post(url, data={'test2': 'test2'}).text)
self.assertIn('test2', r2['form'])
req = Request('POST', url).prepare()
self.assert_(not self.s.cache.has_key(self.s.cache.create_key(req)))
def test_disabled(self):
url = httpbin('get')
requests_cache.install_cache(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE)
requests.get(url)
with requests_cache.disabled():
for i in range(2):
r = requests.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
with self.s.cache_disabled():
for i in range(2):
r = self.s.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
r = self.s.get(url)
self.assertTrue(getattr(r, 'from_cache', False))
def test_enabled(self):
url = httpbin('get')
options = dict(cache_name=CACHE_NAME, backend=CACHE_BACKEND,
fast_save=FAST_SAVE)
with requests_cache.enabled(**options):
r = requests.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
for i in range(2):
r = requests.get(url)
self.assertTrue(getattr(r, 'from_cache', False))
r = requests.get(url)
self.assertFalse(getattr(r, 'from_cache', False))
def test_content_and_cookies(self):
requests_cache.install_cache(CACHE_NAME, CACHE_BACKEND)
s = requests.session()
def js(url):
return json.loads(s.get(url).text)
r1 = js(httpbin('cookies/set/test1/test2'))
with requests_cache.disabled():
r2 = js(httpbin('cookies'))
self.assertEqual(r1, r2)
r3 = js(httpbin('cookies'))
with requests_cache.disabled():
r4 = js(httpbin('cookies/set/test3/test4'))
# from cache
self.assertEqual(r3, js(httpbin('cookies')))
# updated
with requests_cache.disabled():
self.assertEqual(r4, js(httpbin('cookies')))
def test_response_history(self):
r1 = self.s.get(httpbin('redirect/3'))
def test_redirect_history(url):
r2 = self.s.get(url)
self.assertTrue(r2.from_cache)
for r11, r22 in zip(r1.history, r2.history):
self.assertEqual(r11.url, r22.url)
test_redirect_history(httpbin('redirect/3'))
test_redirect_history(httpbin('redirect/2'))
r3 = requests.get(httpbin('redirect/1'))
self.assertEqual(len(r3.history), 1)
def test_response_history_simple(self):
r1 = self.s.get(httpbin('redirect/2'))
r2 = self.s.get(httpbin('redirect/1'))
self.assertTrue(r2.from_cache)
def post(self, data):
return json.loads(self.s.post(httpbin('post'), data=data).text)
def test_post_params(self):
# issue #2
self.s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('GET', 'POST'))
d = {'param1': 'test1'}
for _ in range(2):
self.assertEqual(self.post(d)['form'], d)
d = {'param1': 'test1', 'param3': 'test3'}
self.assertEqual(self.post(d)['form'], d)
self.assertTrue(self.s.post(httpbin('post'), data=d).from_cache)
d.update({'something': 'else'})
self.assertFalse(self.s.post(httpbin('post'), data=d).from_cache)
def test_post_data(self):
# issue #2, raw payload
self.s = CachedSession(CACHE_NAME, CACHE_BACKEND,
allowable_methods=('GET', 'POST'))
d1 = json.dumps({'param1': 'test1'})
d2 = json.dumps({'param1': 'test1', 'param2': 'test2'})
d3 = str('some unicode data')
if is_py3:
bin_data = bytes('some binary data', 'utf8')
else:
bin_data = bytes('some binary data')
for d in (d1, d2, d3):
self.assertEqual(self.post(d)['data'], d)
r = self.s.post(httpbin('post'), data=d)
self.assert_(hasattr(r, 'from_cache'))
self.assertEqual(self.post(bin_data)['data'],
bin_data.decode('utf8'))
r = self.s.post(httpbin('post'), data=bin_data)
self.assert_(hasattr(r, 'from_cache'))
def test_get_params_as_argument(self):
for _ in range(5):
p = {'arg1': 'value1'}
r = self.s.get(httpbin('get'), params=p)
self.assert_(self.s.cache.has_url( httpbin('get?arg1=value1')))
def test_https_support(self):
n = 10
delay = 1
url = 'https://httpbin.org/delay/%s?ar1=value1' % delay
t = time.time()
for _ in range(n):
r = self.s.get(url, verify=False)
self.assertLessEqual(time.time() - t, delay * n / 2)
def test_from_cache_attribute(self):
url = httpbin('get?q=1')
self.assertFalse(self.s.get(url).from_cache)
self.assertTrue(self.s.get(url).from_cache)
self.s.cache.clear()
self.assertFalse(self.s.get(url).from_cache)
def test_gzip_response(self):
url = httpbin('gzip')
self.assertFalse(self.s.get(url).from_cache)
self.assertTrue(self.s.get(url).from_cache)
def request(self,*args,**kwargs) : kwargs.update(self._init["kwargs"]) return CachedSession.request(self,*args,**kwargs)
def test_expire_cache_override_value_error(self):
url = httpbin('delay/1')
s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=1)
with self.assertRaises(ValueError):
s.expire_after(url, 2)
def setUp(self):
self.s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE)
self.s.cache.clear()
requests_cache.uninstall_cache()
