def url_get(url): """ Download an URL using a cache and return the response object :param url: :return: """ s = None info = dict() if 'file://' in url: s = requests.session() s.mount('file://', FileAdapter()) else: s = CachedSession(cache_name="pyff_cache", backend=config.request_cache_backend, expire_after=config.request_cache_time, old_data_on_error=True) headers = {'User-Agent': "pyFF/{}".format(__version__), 'Accept': '*/*'} try: r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout) except IOError as ex: s = requests.Session() r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout) if six.PY2: r.encoding = "utf-8" log.debug("url_get({}) returns {} chrs encoded as {}".format(url, len(r.content), r.encoding)) if config.request_override_encoding is not None: r.encoding = config.request_override_encoding return r
def test_return_old_data_on_error(self, datetime_mock): datetime_mock.utcnow.return_value = datetime.utcnow() expire_after = 100 url = httpbin("get") s = CachedSession(CACHE_NAME, CACHE_BACKEND, old_data_on_error=True, expire_after=expire_after) header = "X-Tst" def get(n): return s.get(url, headers={header: n}).json()["headers"][header] get("expired") self.assertEquals(get("2"), "expired") datetime_mock.utcnow.return_value = datetime.utcnow() + timedelta(seconds=expire_after * 2) with mock.patch.object(s.cache, "save_response", side_effect=Exception): self.assertEquals(get("3"), "expired") with mock.patch("requests_cache.core.OriginalSession.send") as send_mock: resp_mock = requests.Response() request = requests.Request("GET", url) resp_mock.request = request.prepare() resp_mock.status_code = 400 resp_mock._content = '{"other": "content"}' send_mock.return_value = resp_mock self.assertEquals(get("3"), "expired") resp_mock.status_code = 200 self.assertIs(s.get(url).content, resp_mock.content) # default behaviour datetime_mock.return_value = datetime.utcnow() + timedelta(seconds=expire_after * 2) s = CachedSession(CACHE_NAME, CACHE_BACKEND, old_data_on_error=False, expire_after=100) with mock.patch.object(s.cache, "save_response", side_effect=Exception): with self.assertRaises(Exception): s.get(url)
def test_return_old_data_on_error(self, datetime_mock): datetime_mock.utcnow.return_value = datetime.utcnow() expire_after = 100 url = httpbin("get") s = CachedSession(CACHE_NAME, CACHE_BACKEND, old_data_on_error=True, expire_after=expire_after) header = "X-Tst" def get(n): return s.get(url, headers={header: n}).json()["headers"][header] get("expired") self.assertEquals(get("2"), "expired") datetime_mock.utcnow.return_value = datetime.utcnow() + timedelta(seconds=expire_after * 2) with mock.patch.object(s.cache, "save_response", side_effect=Exception): self.assertEquals(get("3"), "expired") with mock.patch("requests_cache.core.OriginalSession.send") as send_mock: resp_mock = requests.Response() request = requests.Request("GET", url) resp_mock.request = request.prepare() resp_mock.status_code = 400 resp_mock._content = '{"other": "content"}' send_mock.return_value = resp_mock self.assertEquals(get("3"), "expired") resp_mock.status_code = 200 self.assertIs(s.get(url).content, resp_mock.content) # default behaviour datetime_mock.return_value = datetime.utcnow() + timedelta(seconds=expire_after * 2) s = CachedSession(CACHE_NAME, CACHE_BACKEND, old_data_on_error=False, expire_after=100) with mock.patch.object(s.cache, "save_response", side_effect=Exception): with self.assertRaises(Exception): s.get(url)
def __init__(self): self.settings = EsiSettings.getInstance() self.server_base: ApiBase = supported_servers[self.settings.get("server")] # session request stuff self._session = Session() self._basicHeaders = { 'Accept': 'application/json', 'User-Agent': ( 'pyfa v{}'.format(config.version) ) } self._session.headers.update(self._basicHeaders) self._session.proxies = NetworkSettings.getInstance().getProxySettingsInRequestsFormat() # Set up cached session. This is only used for SSO meta data for now, but can be expanded to actually handle # various ESI caching (using ETag, for example) in the future cached_session = CachedSession( os.path.join(config.savePath, config.ESI_CACHE), backend="sqlite", cache_control=True, # Use Cache-Control headers for expiration, if available expire_after=timedelta(days=1), # Otherwise expire responses after one day stale_if_error=True, # In case of request errors, use stale cache data if possible ) cached_session.headers.update(self._basicHeaders) cached_session.proxies = NetworkSettings.getInstance().getProxySettingsInRequestsFormat() meta_call = cached_session.get("https://%s/.well-known/oauth-authorization-server" % self.server_base.sso) meta_call.raise_for_status() self.server_meta = meta_call.json() jwks_call = cached_session.get(self.server_meta["jwks_uri"]) jwks_call.raise_for_status() self.jwks = jwks_call.json()
def get_page_url(): """ Initial function to get the list of page urls with respect to different alphabet. :return: list of urls """ page_url_list = [] url = "http://devri.bzh/dictionnaire/a/" session = CachedSession() page = session.get(url) soup = BeautifulSoup(page.content, "html.parser") letter_list = soup.find_all("a", class_="enfant") for i in tqdm(letter_list): url_letter = "http://devri.bzh" + i["href"] page_letter = session.get(url_letter) soup_letter = BeautifulSoup(page_letter.content, "html.parser") page_num = soup_letter.find("li", class_=["MarkupPagerNavLast MarkupPagerNavLastNum", "MarkupPagerNavLastNum"]) try: page_num = page_num.a.text url_list = [url_letter] for j in range(2, int(page_num) + 1): url_list.append(url_letter + f"page{j}") page_url_list += url_list except AttributeError: pass # add page for alphabet z manually page_url_list += ["http://devri.bzh/dictionnaire/z/"] return page_url_list
def test_throttle_cache(self): url = httpbin('get') s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=0.06) s.throttle(url, 0.5) # one every 2 seconds r = s.get(url) time.sleep(0.6) t = time.time() r = s.get(url) delta = time.time() - t self.assertGreaterEqual(delta, 1)
def test_ignore_cgi_parameter(self): s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=5) s.ignore_cgi('foo') url = httpbin('get?foo=1') r = s.get(url) self.assertFalse(r.from_cache) r = s.get(url) self.assertTrue(r.from_cache) url = httpbin('get') r = s.get(url) self.assertTrue(r.from_cache)
def url_get(url: str) -> Response: """ Download an URL using a cache and return the response object :param url: :return: """ s: Union[Session, CachedSession] if 'file://' in url: s = requests.session() s.mount('file://', FileAdapter()) elif 'dir://' in url: s = requests.session() s.mount('dir://', DirAdapter()) else: retry = Retry(total=3, backoff_factor=0.5) adapter = HTTPAdapter(max_retries=retry) s = CachedSession( cache_name="pyff_cache", backend=config.request_cache_backend, expire_after=config.request_cache_time, old_data_on_error=True, ) s.mount('http://', adapter) s.mount('https://', adapter) headers = {'User-Agent': "pyFF/{}".format(__version__), 'Accept': '*/*'} _etag = None if _etag is not None: headers['If-None-Match'] = _etag try: r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout) except IOError as ex: s = requests.Session() r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout) if six.PY2: r.encoding = "utf-8" log.debug("url_get({}) returns {} chrs encoded as {}".format( url, len(r.content), r.encoding)) if config.request_override_encoding is not None: r.encoding = config.request_override_encoding return r
def test_expire_cache(self): delay = 1 url = httpbin('delay/%s' % delay) s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=0.06) t = time.time() r = s.get(url) delta = time.time() - t self.assertGreaterEqual(delta, delay) time.sleep(0.5) t = time.time() r = s.get(url) delta = time.time() - t self.assertGreaterEqual(delta, delay)
def test_expire_cache(self): delay = 1 url = httpbin('delay/%s' % delay) s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=0.06) t = time.time() r = s.get(url) delta = time.time() - t self.assertGreaterEqual(delta, delay) time.sleep(0.5) t = time.time() r = s.get(url) delta = time.time() - t self.assertGreaterEqual(delta, delay)
def test_ignore_parameters(self): url = httpbin("get") ignored_param = "ignored" usual_param = "some" s = CachedSession(CACHE_NAME, CACHE_BACKEND, ignored_parameters=[ignored_param]) params = {ignored_param: "1", usual_param: "1"} s.get(url, params=params) self.assertTrue(s.get(url, params=params).from_cache) params[ignored_param] = "new" self.assertTrue(s.get(url, params=params).from_cache) params[usual_param] = "new" self.assertFalse(s.get(url, params=params).from_cache)
def imdb_search(query): session = CachedSession(expire_after=60 * 60 * 24) stripped_query = "".join([x for x in query if x.isalnum() or x == " "]).lower() r = session.get("https://www.imdb.com/find", params={ "q": stripped_query, "s": "tt", "ttype": "ft" }) page_html = r.text soup = BeautifulSoup(page_html, "html.parser") results = [] for result in soup.select(".result_text"): title = result.a.get_text() imdb_id = result.a.get("href").split("/") try: imdb_id = result.a.get("href").split("/title/")[1].split("/")[0] except IndexError: imdb_id = None continue result.a.decompose() try: year = result.get_text().split("(")[1].split(")")[0] int(year) except: year = None continue results.append((title, year, imdb_id)) return results
def __new__(self, course_id: int, session: CachedSession) -> Tuple[Dict]: lesson_list_at_somewhere_response: CachedResponse = session.get( f"https://foxford.ru/api/courses/{course_id}/lessons", headers={"X-Requested-With": "XMLHttpRequest"}) if lesson_list_at_somewhere_response.status_code != 200: return {"fatal_error": "Lesson list fetch has failed"} if not {"lessons", "cursors"}.issubset( set(lesson_list_at_somewhere_response.json())): return {"fatal_error": "Lesson list structure is unknown"} if "id" not in lesson_list_at_somewhere_response.json()["lessons"][0]: return {"fatal_error": "Lesson structure is unknown"} self.course_id = course_id self.session = session return pipe( lambda json: (*self.recursive_collection(self, "before", json["cursors"][ "before"]), *json["lessons"], *self.recursive_collection( self, "after", json["cursors"]["after"])), lambda lessons: map( lambda lesson: self.lesson_extension(self, lesson), lessons), tuple)(lesson_list_at_somewhere_response.json())
def session_obj(self): session = CachedSession(allowable_methods=('GET', 'POST'), ignored_parameters=['smpbss']) if not IceDaily.RECAPTCHA: with session.cache_disabled(): response = session.get( url= 'https://www.theice.com/marketdata/reports/datawarehouse/ConsolidatedEndOfDayReportPDF.shtml', headers={ 'User-Agent': 'Mozilla/5.0', 'X-Requested-With': 'XMLHttpRequest' }, params={ 'selectionForm': '', 'exchangeCode': 'IFEU', 'optionRequest': self.flavor['optionRequest'] }) bs = BeautifulSoup(response.text) df = pd.DataFrame([(opt['value'], opt.text) for opt in bs.find_all('option')], columns=["WebActiveCode", "ActiveName"]) df['ActiveCode'] = df.WebActiveCode.apply( lambda s: s.split('|', 1)[1] if '|' in s else None) df = df.dropna(how='any') self.update_actives(df) return session
def url_get(url): """ Download an URL using a cache and return the response object :param url: :return: """ s = None info = dict() log.debug("GET URL {!s}".format(url)) if 'file://' in url: s = requests.session() s.mount('file://', FileAdapter()) else: s = CachedSession(cache_name="pyff_cache", backend=config.request_cache_backend, expire_after=config.request_cache_time, old_data_on_error=True) headers = {'User-Agent': "pyFF/{}".format(__version__), 'Accept': '*/*'} r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout) if config.request_override_encoding is not None: r.encoding = config.request_override_encoding return r
def wrapper(self, *args, **kwargs): if not self.omdb_json: session = CachedSession(expires_after=60*60*24*7) if self.imdb_id: r = session.get("http://www.omdbapi.com/", params={ "apiKey": config.OMDB_API_KEY, "i": self.imdb_id }) self.omdb_json = r.json() else: r = session.get("http://www.omdbapi.com/", params={ "apiKey": config.OMDB_API_KEY, "t": self.title, "y": self.year }) self.omdb_json = r.json() return f(self, self.omdb_json, *args, **kwargs)
def test_basic_response_config(self): """ Here is how to load something into a session cache with requests_cache and responses """ session = CachedSession(backend='memory', expire_after=100000) url = 'http://example.com/123' data = {'foo': 'yes please'} responses.add(responses.GET, url, json=data) response_one = session.get(url) self.assertFalse(response_one.from_cache) responses.reset() responses.add(responses.GET, url, json={}) response_two = session.get(url) self.assertTrue(response_two.from_cache) self.assertEqual(response_two.json()['foo'], data['foo'])
def main(): session = CachedSession('example_cache', backend='sqlite') # The real request will only be made once; afterward, the cached response is used for i in range(5): response = session.get('http://httpbin.org/get') # This is more obvious when calling a slow endpoint for i in range(5): response = session.get('http://httpbin.org/delay/2') # Caching can be disabled if we want to get a fresh page and not cache it with session.cache_disabled(): print(session.get('http://httpbin.org/ip').text) # Get some debugging info about the cache print(session.cache) print('Cached URLS:', session.cache.urls)
def wrapper(page_html=None, imdb_id=None, soup=None, *args, **kwargs): if not (page_html or imdb_id or soup): return None if imdb_id: session = CachedSession(expire_after=60 * 60 * 24) r = session.get("https://www.imdb.com/title/{}".format(imdb_id)) page_html = r.text soup = soup or BeautifulSoup(page_html, "html.parser") return f(soup, *args, **kwargs)
def get_data(leaderboard, year): lb_path = LEADERBOARDS.get(leaderboard, leaderboard) url = f"https://adventofcode.com/{year}/leaderboard/private/view/{lb_path}" cache_options = {"backend": "filesystem", "expire_after": 86400} session = CachedSession("../../site_cache", **cache_options) cookies = dict([Path("../.session-cookie").read_text().strip().split("=")]) response = session.get(url, cookies=cookies) if response.status_code != 200: pdb.set_trace() return response.json()
def get_page_content(url: str) -> str: """ Utilization function to get page content literally :param url: string :return: string """ session = CachedSession() page = session.get(url) soup = BeautifulSoup(page.content, "html.parser") content = soup.prettify() return content
def test_ignore_parameters_get(self): url = httpbin("get") ignored_param = "ignored" usual_param = "some" params = {ignored_param: "1", usual_param: "1"} s = CachedSession(CACHE_NAME, CACHE_BACKEND, ignored_parameters=[ignored_param]) r = s.get(url, params=params) self.assertIn(ignored_param, r.json()['args'].keys()) self.assertFalse(r.from_cache) self.assertTrue(s.get(url, params=params).from_cache) params[ignored_param] = "new" self.assertTrue(s.get(url, params=params).from_cache) params[usual_param] = "new" self.assertFalse(s.get(url, params=params).from_cache)
def fetch_df(url): r = requests.get(url) js = r.json() if "features" not in js: s = CachedSession() with s.cache_disabled(): r = s.get(url) js = r.json() features = [feature["attributes"] for feature in js["features"]] df = pd.DataFrame(features) return df
def test_ignore_parameters_get(self): url = httpbin("get") ignored_param = "ignored" usual_param = "some" params = {ignored_param: "1", usual_param: "1"} s = CachedSession(CACHE_NAME, CACHE_BACKEND, ignored_parameters=[ignored_param]) r = s.get(url, params=params) self.assertIn(ignored_param, r.json()['args'].keys()) self.assertFalse(r.from_cache) self.assertTrue(s.get(url, params=params).from_cache) params[ignored_param] = "new" self.assertTrue(s.get(url, params=params).from_cache) params[usual_param] = "new" self.assertFalse(s.get(url, params=params).from_cache)
def get(self, resource: str, params=None): client = BackendApplicationClient(client_id=self.client_id) oauth = OAuth2Session(client=client) token = oauth.fetch_token(token_url='https://us.battle.net/oauth/token', client_id=self.client_id, client_secret=self.client_secret) if params is None: params = {} root = 'https://us.api.blizzard.com/' params.update({'locale': 'en_US', 'namespace': 'static-us', 'access_token': token['access_token']}) session = CachedSession(expire_after=timedelta(hours=1)) return session.get(root + resource, params=params).json()
def test_loader_with_session(self): session = CachedSession(backend='memory', expire_after=100000) loadurl = CachableDocumentLoader(use_cache=True, session=session) assertion_data = json.loads(test_components['2_0_basic_assertion']) context_url = assertion_data['@context'] set_up_context_mock() session.get(context_url) # precache response responses.reset() responses.add(responses.GET, context_url, json={'nothing': 'happenin'}) first_compacted = jsonld.compact(assertion_data, context_url, options={'documentLoader': loadurl}) second_compacted = jsonld.compact(assertion_data, context_url, options={'documentLoader': loadurl}) # second compaction should have built from the cache self.assertEqual(first_compacted['verification']['type'], second_compacted['verification']['type'])
class AstronomyPictureOfThedaySkill(MycroftSkill): def __init__(self): super(AstronomyPictureOfThedaySkill, self).__init__(name="AstronomyPictureOfTheday") if "nasa_key" not in self.settings: self.settings["nasa_key"] = "DEMO_KEY" _expire_after = timedelta(hours=1) self._session = CachedSession(backend='memory', expire_after=_expire_after) def update_picture(self): try: apod_url = "https://api.nasa.gov/planetary/apod?api_key=" + self.settings[ "nasa_key"] response = self._session.get(apod_url).json() title = response["title"] url = response["url"] summary = response["explanation"] if not self.lang.lower().startswith("en"): summary = translate(summary, self.lang) title = translate(title, self.lang) self.settings['imgLink'] = url self.settings['title'] = title self.settings['summary'] = summary except Exception as e: self.log.exception(e) self.gui['imgLink'] = self.settings['imgLink'] self.gui['title'] = self.settings['title'] self.gui['summary'] = self.settings['summary'] self.set_context("APOD") @resting_screen_handler("APOD") def idle(self, message): self.update_picture() self.gui.clear() self.gui.show_page('idle.qml') @intent_file_handler('apod.intent') def handle_apod(self, message): self.update_picture() self.gui.clear() self.gui.show_image(self.settings['imgLink'], caption=self.settings['title'], fill='PreserveAspectFit') self.speak(self.settings['title']) @intent_handler( IntentBuilder("ExplainIntent").require("ExplainKeyword").require( "APOD")) def handle_explain(self, message): self.speak(self.settings['summary'])
def get_csrf_token(session: CachedSession) -> str: csrf_token_get_response: CachedResponse = session.get( "https://foxford.ru/api/csrf_token", headers={"X-Requested-With": "XMLHttpRequest"}) if csrf_token_get_response.status_code != 200: return {"fatal_error": "CSRF token fetch has failed"} if "token" not in csrf_token_get_response.json(): return {"fatal_error": "CSRF token structure is unknown"} return csrf_token_get_response.json()["token"]
def main(): session = CachedSession('example_cache', backend='sqlite') # By default, cached responses never expire response = session.get('https://httpbin.org/get') assert not response.from_cache response = session.get('https://httpbin.org/get') assert response.from_cache assert not response.expires # We can set default expiration for the session using expire_after session = CachedSession('example_cache', backend='sqlite', expire_after=60) session.cache.clear() response = session.get('https://httpbin.org/get') response = session.get('https://httpbin.org/get') print('Expiration time:', response.expires) # This can also be overridden for individual requests session.cache.clear() response = session.get('https://httpbin.org/get', expire_after=1) response = session.get('https://httpbin.org/get') assert response.from_cache print('Expiration time:', response.expires) # After 1 second, the cached value will expired time.sleep(1.2) assert response.is_expired response = session.get('https://httpbin.org/get') assert not response.from_cache
def url_get(url): """ Download an URL using a cache and return the response object :param url: :return: """ s = None info = dict() if 'file://' in url: s = requests.session() s.mount('file://', FileAdapter()) else: s = CachedSession(cache_name="pyff_cache", backend=config.request_cache_backend, expire_after=config.request_cache_time, old_data_on_error=True) headers = {'User-Agent': "pyFF/{}".format(__version__), 'Accept': '*/*'} try: r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout) except IOError as ex: s = requests.Session() r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout) if six.PY2: r.encoding = "utf-8" log.debug("url_get({}) returns {} chrs encoded as {}".format( url, len(r.content), r.encoding)) if config.request_override_encoding is not None: r.encoding = config.request_override_encoding return r
def wrapper(title, year, *args, **kwargs): session = CachedSession(expire_after=60 * 60 * 24) # https://www.rottentomatoes.com/napi/search/?query=parasite&offset=0&limit=10 stripped_title = "".join([x for x in title if x.isalnum() or x == " "]).lower().strip() response = session.get("https://www.rottentomatoes.com/napi/search", params={ "query": stripped_title, "offset": 0, "limit": 10 }) json = response.json() return f(stripped_title, year, json, *args, **kwargs)
def get_profile(request, provider_name): provider = SOCIAL_AUTH_PROVIDERS[provider_name]() # Prepare the session for fetching the token. session = OAuth2Session( client_id=provider.client_id, scope=provider.scope, state=request.session.get('social_login_state', ''), redirect_uri=request.build_absolute_uri( reverse('social_login_callback', kwargs={'provider_name': provider_name})), ) # Clear the session state data before continuing. if 'social_login_state' in request.session: del request.session['social_login_state'] # Go and fetch the oauth token. token = session.fetch_token( token_url=provider.token_uri, client_secret=provider.client_secret, authorization_response=request.build_absolute_uri()) # Get the id_token from the oauth token. unparsed_id_token = token['id_token'] # Retrieve the certificates from the provider, this is in json format and is cached for 1 hour. expire_after = datetime.timedelta(hours=1) cached_session = CachedSession(backend='memory', expire_after=expire_after) provider_certificates = cached_session.get( provider.jwks_uri).json().get('keys') # Put the certificates in a dict, with the identifier as key. certificate_set = {cert['kid']: cert for cert in provider_certificates} # Look up which certificate was used to sign the id_token. kid = jwt.get_unverified_header(unparsed_id_token).get('kid') # Convert the certificate from json to something the jwt library can use. certificate = RSAAlgorithm.from_jwk(json.dumps(certificate_set[kid])) # Now finally do the actual decoding and verifying. id_token = jwt.decode(unparsed_id_token, certificate, audience=provider.client_id) # Put the parsed id_token in the response. token['id_token'] = id_token return provider.parse_profile(session, token)
def get_entry_url(url: str) -> list: """ Function to extract url of entries from alphabet page url :param url: string :return: list of urls """ url_list = [] session = CachedSession() page = session.get(url) soup = BeautifulSoup(page.content, "html.parser") entry_url = soup.find_all("li", class_="list-group-item col-md-4") for i in tqdm(entry_url): url = "http://devri.bzh" + i.a["href"] url_list.append(url) return url_list
def test_headers_in_get_query(self): url = httpbin("get") s = CachedSession(CACHE_NAME, CACHE_BACKEND, include_get_headers=True) headers = {"Accept": "text/json"} self.assertFalse(s.get(url, headers=headers).from_cache) self.assertTrue(s.get(url, headers=headers).from_cache) headers["Accept"] = "text/xml" self.assertFalse(s.get(url, headers=headers).from_cache) self.assertTrue(s.get(url, headers=headers).from_cache) headers["X-custom-header"] = "custom" self.assertFalse(s.get(url, headers=headers).from_cache) self.assertTrue(s.get(url, headers=headers).from_cache) self.assertFalse(s.get(url).from_cache) self.assertTrue(s.get(url).from_cache)
def api(url,alternate_domain="",cache:CachedSession=None): log("Appel de "+url) if cache: data=cache.get(url) else: data=requests.api.get(url) if data.status_code!=200: log("Appel de "+url) url=url.replace(alternate_domain.split("=")[0],alternate_domain.split("=")[1]) if cache: data=cache.get(url) else: data=requests.api.get(url) if data.status_code!=200: log("Echec de l'appel "+str(data.status_code)+" "+data.text) return None try: return data.json() except: return data.text
def test_headers_in_get_query(self): url = httpbin("get") s = CachedSession(CACHE_NAME, CACHE_BACKEND, include_get_headers=True) headers = {"Accept": "text/json"} self.assertFalse(s.get(url, headers=headers).from_cache) self.assertTrue(s.get(url, headers=headers).from_cache) headers["Accept"] = "text/xml" self.assertFalse(s.get(url, headers=headers).from_cache) self.assertTrue(s.get(url, headers=headers).from_cache) headers["X-custom-header"] = "custom" self.assertFalse(s.get(url, headers=headers).from_cache) self.assertTrue(s.get(url, headers=headers).from_cache) self.assertFalse(s.get(url).from_cache) self.assertTrue(s.get(url).from_cache)
def api_get(endpoint, query, cache=False): if not cache: s = CachedSession() with s.cache_disabled(): r = s.get( app.config['API_URL'] + endpoint + '?q=' + urllib.parse.quote_plus(json.dumps(query)), headers=gen_api_header(), verify=app.config['VERIFY_SSL']) else: r = requests.get( app.config['API_URL'] + endpoint + '?q=' + urllib.parse.quote_plus(json.dumps(query)), headers=gen_api_header(), verify=app.config['VERIFY_SSL']) if r.status_code == 200: # If created then it returns the object data return json.loads(r.text).get('objects') else: return {}
def test_remove_expired_entries(self, datetime_mock, datetime_mock2): expire_after = timedelta(minutes=10) start_time = datetime.utcnow().replace(year=2010, minute=0) datetime_mock.utcnow.return_value = start_time datetime_mock2.utcnow.return_value = start_time s = CachedSession(CACHE_NAME, CACHE_BACKEND, expire_after=expire_after) s.get(httpbin('get')) s.get(httpbin('relative-redirect/3')) datetime_mock.utcnow.return_value = start_time + expire_after * 2 datetime_mock2.utcnow.return_value = datetime_mock.utcnow.return_value ok_url = 'get?x=1' s.get(httpbin(ok_url)) self.assertEqual(len(s.cache.responses), 3) self.assertEqual(len(s.cache.keys_map), 3) s.remove_expired_responses() self.assertEqual(len(s.cache.responses), 1) self.assertEqual(len(s.cache.keys_map), 0) self.assertIn(ok_url, list(s.cache.responses.values())[0][0].url)
#!/usr/bin/env python # -*- coding: utf-8 -*- import requests from requests_cache import CachedSession cs = CachedSession(allowable_methods=('GET', 'POST')) cs.cache.clear() for i in range(2): r = cs.get("http://httpbin.org/get?p1=v1", params={'p2': 'v2', 'p3': 'cyrЯЯ'}) print r print r.from_cache
class CacheTestCase(unittest.TestCase): def setUp(self): self.s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE) self.s.cache.clear() requests_cache.uninstall_cache() def test_expire_cache(self): delay = 1 url = httpbin('delay/%s' % delay) s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=0.06) t = time.time() r = s.get(url) delta = time.time() - t self.assertGreaterEqual(delta, delay) time.sleep(0.5) t = time.time() r = s.get(url) delta = time.time() - t self.assertGreaterEqual(delta, delay) def test_delete_urls(self): url = httpbin('relative-redirect/3') r = self.s.get(url) for i in range(1, 4): self.assert_(self.s.cache.has_url(httpbin('relative-redirect/%s' % i))) self.s.cache.delete_url(url) self.assert_(not self.s.cache.has_url(url)) def test_unregistered_backend(self): with self.assertRaises(ValueError): CachedSession(CACHE_NAME, backend='nonexistent') def test_hooks(self): state = defaultdict(int) for hook in ('response',): # TODO it's only one hook here def hook_func(r, *args, **kwargs): state[hook] += 1 return r n = 5 for i in range(n): r = self.s.get(httpbin('get'), hooks={hook: hook_func}) self.assertEqual(state[hook], n) def test_attr_from_cache_in_hook(self): state = defaultdict(int) hook = 'response' def hook_func(r, *args, **kwargs): if state[hook] > 0: self.assert_(r.from_cache, True) state[hook] += 1 return r n = 5 for i in range(n): r = self.s.get(httpbin('get'), hooks={hook: hook_func}) self.assertEqual(state[hook], n) def test_post(self): url = httpbin('post') r1 = json.loads(self.s.post(url, data={'test1': 'test1'}).text) r2 = json.loads(self.s.post(url, data={'test2': 'test2'}).text) self.assertIn('test2', r2['form']) req = Request('POST', url).prepare() self.assert_(not self.s.cache.has_key(self.s.cache.create_key(req))) def test_disabled(self): url = httpbin('get') requests_cache.install_cache(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE) requests.get(url) with requests_cache.disabled(): for i in range(2): r = requests.get(url) self.assertFalse(getattr(r, 'from_cache', False)) with self.s.cache_disabled(): for i in range(2): r = self.s.get(url) self.assertFalse(getattr(r, 'from_cache', False)) r = self.s.get(url) self.assertTrue(getattr(r, 'from_cache', False)) def test_enabled(self): url = httpbin('get') options = dict(cache_name=CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE) with requests_cache.enabled(**options): r = requests.get(url) self.assertFalse(getattr(r, 'from_cache', False)) for i in range(2): r = requests.get(url) self.assertTrue(getattr(r, 'from_cache', False)) r = requests.get(url) self.assertFalse(getattr(r, 'from_cache', False)) def test_content_and_cookies(self): requests_cache.install_cache(CACHE_NAME, CACHE_BACKEND) s = requests.session() def js(url): return json.loads(s.get(url).text) r1 = js(httpbin('cookies/set/test1/test2')) with requests_cache.disabled(): r2 = js(httpbin('cookies')) self.assertEqual(r1, r2) r3 = js(httpbin('cookies')) with requests_cache.disabled(): r4 = js(httpbin('cookies/set/test3/test4')) # from cache self.assertEqual(r3, js(httpbin('cookies'))) # updated with requests_cache.disabled(): self.assertEqual(r4, js(httpbin('cookies'))) def test_response_history(self): r1 = self.s.get(httpbin('relative-redirect/3')) def test_redirect_history(url): r2 = self.s.get(url) self.assertTrue(r2.from_cache) for r11, r22 in zip(r1.history, r2.history): self.assertEqual(r11.url, r22.url) test_redirect_history(httpbin('relative-redirect/3')) test_redirect_history(httpbin('relative-redirect/2')) r3 = requests.get(httpbin('relative-redirect/1')) self.assertEqual(len(r3.history), 1) def test_response_history_simple(self): r1 = self.s.get(httpbin('relative-redirect/2')) r2 = self.s.get(httpbin('relative-redirect/1')) self.assertTrue(r2.from_cache) def post(self, data): return json.loads(self.s.post(httpbin('post'), data=data).text) def test_post_params(self): # issue #2 self.s = CachedSession(CACHE_NAME, CACHE_BACKEND, allowable_methods=('GET', 'POST')) d = {'param1': 'test1'} for _ in range(2): self.assertEqual(self.post(d)['form'], d) d = {'param1': 'test1', 'param3': 'test3'} self.assertEqual(self.post(d)['form'], d) self.assertTrue(self.s.post(httpbin('post'), data=d).from_cache) d.update({'something': 'else'}) self.assertFalse(self.s.post(httpbin('post'), data=d).from_cache) def test_post_data(self): # issue #2, raw payload self.s = CachedSession(CACHE_NAME, CACHE_BACKEND, allowable_methods=('GET', 'POST')) d1 = json.dumps({'param1': 'test1'}) d2 = json.dumps({'param1': 'test1', 'param2': 'test2'}) d3 = str('some unicode data') if is_py3: bin_data = bytes('some binary data', 'utf8') else: bin_data = bytes('some binary data') for d in (d1, d2, d3): self.assertEqual(self.post(d)['data'], d) r = self.s.post(httpbin('post'), data=d) self.assert_(hasattr(r, 'from_cache')) self.assertEqual(self.post(bin_data)['data'], bin_data.decode('utf8')) r = self.s.post(httpbin('post'), data=bin_data) self.assert_(hasattr(r, 'from_cache')) def test_get_params_as_argument(self): for _ in range(5): p = {'arg1': 'value1'} r = self.s.get(httpbin('get'), params=p) self.assert_(self.s.cache.has_url( httpbin('get?arg1=value1'))) def test_https_support(self): n = 10 delay = 1 url = 'https://httpbin.org/delay/%s?ar1=value1' % delay t = time.time() for _ in range(n): r = self.s.get(url, verify=False) self.assertLessEqual(time.time() - t, delay * n / 2) def test_from_cache_attribute(self): url = httpbin('get?q=1') self.assertFalse(self.s.get(url).from_cache) self.assertTrue(self.s.get(url).from_cache) self.s.cache.clear() self.assertFalse(self.s.get(url).from_cache) def test_gzip_response(self): url = httpbin('gzip') self.assertFalse(self.s.get(url).from_cache) self.assertTrue(self.s.get(url).from_cache) def test_close_response(self): for _ in range(3): r = self.s.get(httpbin("get")) r.close() def test_get_parameters_normalization(self): url = httpbin("get") params = {"a": "a", "b": ["1", "2", "3"], "c": "4"} self.assertFalse(self.s.get(url, params=params).from_cache) r = self.s.get(url, params=params) self.assertTrue(r.from_cache) self.assertEquals(r.json()["args"], params) self.assertFalse(self.s.get(url, params={"a": "b"}).from_cache) self.assertTrue(self.s.get(url, params=sorted(params.items())).from_cache) class UserSubclass(dict): def items(self): return sorted(super(UserSubclass, self).items(), reverse=True) params["z"] = "5" custom_dict = UserSubclass(params) self.assertFalse(self.s.get(url, params=custom_dict).from_cache) self.assertTrue(self.s.get(url, params=custom_dict).from_cache) def test_post_parameters_normalization(self): params = {"a": "a", "b": ["1", "2", "3"], "c": "4"} url = httpbin("post") s = CachedSession(CACHE_NAME, CACHE_BACKEND, allowable_methods=('GET', 'POST')) self.assertFalse(s.post(url, data=params).from_cache) self.assertTrue(s.post(url, data=params).from_cache) self.assertTrue(s.post(url, data=sorted(params.items())).from_cache) self.assertFalse(s.post(url, data=sorted(params.items(), reverse=True)).from_cache) def test_stream_requests_support(self): n = 100 url = httpbin("stream/%s" % n) r = self.s.get(url, stream=True) lines = list(r.iter_lines()) self.assertEquals(len(lines), n) for i in range(2): r = self.s.get(url, stream=True) self.assertTrue(r.from_cache) cached_lines = list(r.iter_lines()) self.assertEquals(cached_lines, lines) def test_headers_in_get_query(self): url = httpbin("get") s = CachedSession(CACHE_NAME, CACHE_BACKEND, include_get_headers=True) headers = {"Accept": "text/json"} self.assertFalse(s.get(url, headers=headers).from_cache) self.assertTrue(s.get(url, headers=headers).from_cache) headers["Accept"] = "text/xml" self.assertFalse(s.get(url, headers=headers).from_cache) self.assertTrue(s.get(url, headers=headers).from_cache) headers["X-custom-header"] = "custom" self.assertFalse(s.get(url, headers=headers).from_cache) self.assertTrue(s.get(url, headers=headers).from_cache) self.assertFalse(s.get(url).from_cache) self.assertTrue(s.get(url).from_cache) def test_str_and_repr(self): s = repr(CachedSession(CACHE_NAME, CACHE_BACKEND, expire_after=10)) self.assertIn(CACHE_NAME, s) self.assertIn("10", s) @mock.patch("requests_cache.core.datetime") def test_return_old_data_on_error(self, datetime_mock): datetime_mock.utcnow.return_value = datetime.utcnow() expire_after = 100 url = httpbin("get") s = CachedSession(CACHE_NAME, CACHE_BACKEND, old_data_on_error=True, expire_after=expire_after) header = "X-Tst" def get(n): return s.get(url, headers={header: n}).json()["headers"][header] get("expired") self.assertEquals(get("2"), "expired") datetime_mock.utcnow.return_value = datetime.utcnow() + timedelta(seconds=expire_after * 2) with mock.patch.object(s.cache, "save_response", side_effect=Exception): self.assertEquals(get("3"), "expired") with mock.patch("requests_cache.core.OriginalSession.send") as send_mock: resp_mock = requests.Response() request = requests.Request("GET", url) resp_mock.request = request.prepare() resp_mock.status_code = 400 resp_mock._content = '{"other": "content"}' send_mock.return_value = resp_mock self.assertEquals(get("3"), "expired") resp_mock.status_code = 200 self.assertIs(s.get(url).content, resp_mock.content) # default behaviour datetime_mock.return_value = datetime.utcnow() + timedelta(seconds=expire_after * 2) s = CachedSession(CACHE_NAME, CACHE_BACKEND, old_data_on_error=False, expire_after=100) with mock.patch.object(s.cache, "save_response", side_effect=Exception): with self.assertRaises(Exception): s.get(url) def test_ignore_parameters_get(self): url = httpbin("get") ignored_param = "ignored" usual_param = "some" params = {ignored_param: "1", usual_param: "1"} s = CachedSession(CACHE_NAME, CACHE_BACKEND, ignored_parameters=[ignored_param]) r = s.get(url, params=params) self.assertIn(ignored_param, r.json()['args'].keys()) self.assertFalse(r.from_cache) self.assertTrue(s.get(url, params=params).from_cache) params[ignored_param] = "new" self.assertTrue(s.get(url, params=params).from_cache) params[usual_param] = "new" self.assertFalse(s.get(url, params=params).from_cache) def test_ignore_parameters_post(self): url = httpbin("post") ignored_param = "ignored" usual_param = "some" d = {ignored_param: "1", usual_param: "1"} s = CachedSession(CACHE_NAME, CACHE_BACKEND, allowable_methods=('POST'), ignored_parameters=[ignored_param]) r = s.post(url, data=d) self.assertIn(ignored_param, r.json()['form'].keys()) self.assertFalse(r.from_cache) self.assertTrue(s.post(url, data=d).from_cache) d[ignored_param] = "new" self.assertTrue(s.post(url, data=d).from_cache) d[usual_param] = "new" self.assertFalse(s.post(url, data=d).from_cache) def test_ignore_parameters_post_json(self): url = httpbin("post") ignored_param = "ignored" usual_param = "some" d = {ignored_param: "1", usual_param: "1"} s = CachedSession(CACHE_NAME, CACHE_BACKEND, allowable_methods=('POST'), ignored_parameters=[ignored_param]) r = s.post(url, json=d) self.assertIn(ignored_param, json.loads(r.json()['data']).keys()) self.assertFalse(r.from_cache) self.assertTrue(s.post(url, json=d).from_cache) d[ignored_param] = "new" self.assertTrue(s.post(url, json=d).from_cache) d[usual_param] = "new" self.assertFalse(s.post(url, json=d).from_cache) def test_ignore_parameters_post_raw(self): url = httpbin("post") ignored_param = "ignored" raw_data = "raw test data" s = CachedSession(CACHE_NAME, CACHE_BACKEND, allowable_methods=('POST'), ignored_parameters=[ignored_param]) self.assertFalse(s.post(url, data=raw_data).from_cache) self.assertTrue(s.post(url, data=raw_data).from_cache) raw_data = "new raw data" self.assertFalse(s.post(url, data=raw_data).from_cache) @mock.patch("requests_cache.backends.base.datetime") @mock.patch("requests_cache.core.datetime") def test_remove_expired_entries(self, datetime_mock, datetime_mock2): expire_after = timedelta(minutes=10) start_time = datetime.utcnow().replace(year=2010, minute=0) datetime_mock.utcnow.return_value = start_time datetime_mock2.utcnow.return_value = start_time s = CachedSession(CACHE_NAME, CACHE_BACKEND, expire_after=expire_after) s.get(httpbin('get')) s.get(httpbin('relative-redirect/3')) datetime_mock.utcnow.return_value = start_time + expire_after * 2 datetime_mock2.utcnow.return_value = datetime_mock.utcnow.return_value ok_url = 'get?x=1' s.get(httpbin(ok_url)) self.assertEqual(len(s.cache.responses), 3) self.assertEqual(len(s.cache.keys_map), 3) s.remove_expired_responses() self.assertEqual(len(s.cache.responses), 1) self.assertEqual(len(s.cache.keys_map), 0) self.assertIn(ok_url, list(s.cache.responses.values())[0][0].url)
class DownloadExternalResource(object): def __init__(self, url=None, max_file_size=None, allowed_content_types=(), jsfetcher=None, marker=r'<meta.*name="Generator".*>'): logger.info('Running downloader for url: %s' % url) self.url = str(url) self.max_file_size = max_file_size or settings.RFDOCS.get('MAX_FILE_SIZE') self.allowed_content_types = allowed_content_types or settings.RFDOCS.get('ALLOWED_CONTENT_TYPES') self.s = CachedSession() self.r = None self.error = {} self.jsfetcher = jsfetcher self.marker = marker def _set_error(self, key, value): self.error[key] = value return self.error def _no_error(self): self.error = {} return self.error def send_request(self): logger.info('Download external resource: %s' % self.url) try: self.r = self.s.get(self.url) except (requests.exceptions.MissingSchema, requests.exceptions.InvalidSchema, requests.exceptions.ConnectionError,) as error: logger.warn("Failed to download resource. Error: %s" % error) self._set_error('error', error) def get_response(self): # Workaround for Robot Framework libraries with version >= 2.8 (or even 2.7). # Libraries are generated with JQuery templates system. # Python's `requests` module fetches raw content, of course not rendered HTML. # Thus parser will fail in such cases. # For this reason an awesome PhantomJS is used. Unfortunately `requests_cache` module # wont help here because PhantomJS has to request the URL by its # own methods to be able to render HTML. # One more notice is that I don't use PhantomJS native features to actually write data to filesystem. # (PhantomJS has `fs` module to work with filesystem). # Why? I don't know yet :) # Instead use `subprocess.Popen` to execute javascript code, # let PhantomJS to do his job and give the output back to python. # Anyway I want the resource to be validated by Django's validators before we can proceed. # So if we came here, the validation has passed. # First, check if response content has # meta tag with name="Generator" # If so, the document uses JQuery templates plugin system and we need help of PhantomJS. content = self.r.content mo = re.search(self.marker, content, re.DOTALL | re.M | re.I) if not mo: return content logger.info('Using phantomjs to download resource') alternate_downloader = PhantomJSHelper(url=self.url, error_callback=self._set_error) return alternate_downloader.get_content() def validate_response(self): logger.info('Validate external resource: %s' % self.url) if not self.r: self.send_request() if self.error: return self.error try: self.r.raise_for_status() except requests.exceptions.HTTPError as error: logger.warn("Failed to fetch resource. Error: %s" % error) return self._set_error('error', error) if self.r.status_code == requests.codes.ok: content_len = self.r.headers.get('content-length', None) if content_len: csize = int(content_len) else: csize = len(self.r.content) if csize < self.max_file_size: ctype_header = self.r.headers.get('content-type') if not ctype_header: return self._set_error('content_type', 'Response does not contain the \'Content-Type\' header. Rejected.') ctype = ctype_header.split(';')[0].lower() if ctype in [ct.lower() for ct in self.allowed_content_types]: # the place where all the procedure passed and we succeeded return self._no_error() else: logger.warn("Failed to fetch resource. " "Allowed content types are: %s." "The content type is: %s" % (', '.join(self.allowed_content_types), ctype,)) return self._set_error('content_type', ctype) else: logger.warn("Failed to fetch resource. " "The content size \'%s\' exceeds maximum allowed size: %s" % (self.max_file_size, csize)) return self._set_error('content_size', csize) else: error = requests.exceptions.HTTPError(self.r.status_code) logger.warn("Failed to fetch resource. Error: %s" % error) return self._set_error('error', error) def get_response_from_cache(self): if not self.s.cache.has_url(self.url): self.send_request() self.validate_response() return self.get_response() self.r = self.s.get(self.url) return self.get_response() def get_response_from_cache_or_raise_error(self): response = self.get_response_from_cache() if self.error: err = self.error.get('error') # these are model's `clean` ValidationError (not the same as forms.ValidationError) if err: raise exceptions.ValidationError(err) else: raise exceptions.ValidationError(self.error) return response
class CacheTestCase(unittest.TestCase): def setUp(self): self.s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE) self.s.cache.clear() requests_cache.uninstall_cache() def test_expire_cache(self): delay = 1 url = httpbin('delay/%s' % delay) s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, expire_after=0.06) t = time.time() r = s.get(url) delta = time.time() - t self.assertGreaterEqual(delta, delay) time.sleep(0.5) t = time.time() r = s.get(url) delta = time.time() - t self.assertGreaterEqual(delta, delay) def test_delete_urls(self): url = httpbin('redirect/3') r = self.s.get(url) for i in range(1, 4): self.assert_(self.s.cache.has_url(httpbin('redirect/%s' % i))) self.s.cache.delete_url(url) self.assert_(not self.s.cache.has_url(url)) def test_unregistered_backend(self): with self.assertRaises(ValueError): CachedSession(CACHE_NAME, backend='nonexistent') # def test_async_compatibility(self): # try: # import grequests # except Exception: # self.skipTest('gevent is not installed') # n = 3 # def long_running(): # t = time.time() # rs = [grequests.get(httpbin('delay/%s' % i)) for i in range(n + 1)] # grequests.map(rs) # return time.time() - t # # cache it # delta = long_running() # self.assertGreaterEqual(delta, n) # # fast from cache # delta = 0 # for i in range(n): # delta += long_running() # self.assertLessEqual(delta, 1) def test_hooks(self): state = defaultdict(int) for hook in ('response',): # TODO it's only one hook here def hook_func(r, *args, **kwargs): state[hook] += 1 return r n = 5 for i in range(n): r = self.s.get(httpbin('get'), hooks={hook: hook_func}) self.assertEqual(state[hook], n) def test_attr_from_cache_in_hook(self): state = defaultdict(int) hook = 'response' def hook_func(r, *args, **kwargs): if state[hook] > 0: self.assert_(r.from_cache, True) state[hook] += 1 return r n = 5 for i in range(n): r = self.s.get(httpbin('get'), hooks={hook: hook_func}) self.assertEqual(state[hook], n) def test_post(self): url = httpbin('post') r1 = json.loads(self.s.post(url, data={'test1': 'test1'}).text) r2 = json.loads(self.s.post(url, data={'test2': 'test2'}).text) self.assertIn('test2', r2['form']) req = Request('POST', url).prepare() self.assert_(not self.s.cache.has_key(self.s.cache.create_key(req))) def test_disabled(self): url = httpbin('get') requests_cache.install_cache(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE) requests.get(url) with requests_cache.disabled(): for i in range(2): r = requests.get(url) self.assertFalse(getattr(r, 'from_cache', False)) with self.s.cache_disabled(): for i in range(2): r = self.s.get(url) self.assertFalse(getattr(r, 'from_cache', False)) r = self.s.get(url) self.assertTrue(getattr(r, 'from_cache', False)) def test_enabled(self): url = httpbin('get') options = dict(cache_name=CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE) with requests_cache.enabled(**options): r = requests.get(url) self.assertFalse(getattr(r, 'from_cache', False)) for i in range(2): r = requests.get(url) self.assertTrue(getattr(r, 'from_cache', False)) r = requests.get(url) self.assertFalse(getattr(r, 'from_cache', False)) def test_content_and_cookies(self): requests_cache.install_cache(CACHE_NAME, CACHE_BACKEND) s = requests.session() def js(url): return json.loads(s.get(url).text) r1 = js(httpbin('cookies/set/test1/test2')) with requests_cache.disabled(): r2 = js(httpbin('cookies')) self.assertEqual(r1, r2) r3 = js(httpbin('cookies')) with requests_cache.disabled(): r4 = js(httpbin('cookies/set/test3/test4')) # from cache self.assertEqual(r3, js(httpbin('cookies'))) # updated with requests_cache.disabled(): self.assertEqual(r4, js(httpbin('cookies'))) def test_response_history(self): r1 = self.s.get(httpbin('redirect/3')) def test_redirect_history(url): r2 = self.s.get(url) self.assertTrue(r2.from_cache) for r11, r22 in zip(r1.history, r2.history): self.assertEqual(r11.url, r22.url) test_redirect_history(httpbin('redirect/3')) test_redirect_history(httpbin('redirect/2')) r3 = requests.get(httpbin('redirect/1')) self.assertEqual(len(r3.history), 1) def test_response_history_simple(self): r1 = self.s.get(httpbin('redirect/2')) r2 = self.s.get(httpbin('redirect/1')) self.assertTrue(r2.from_cache) def post(self, data): return json.loads(self.s.post(httpbin('post'), data=data).text) def test_post_params(self): # issue #2 self.s = CachedSession(CACHE_NAME, CACHE_BACKEND, allowable_methods=('GET', 'POST')) d = {'param1': 'test1'} for _ in range(2): self.assertEqual(self.post(d)['form'], d) d = {'param1': 'test1', 'param3': 'test3'} self.assertEqual(self.post(d)['form'], d) self.assertTrue(self.s.post(httpbin('post'), data=d).from_cache) d.update({'something': 'else'}) self.assertFalse(self.s.post(httpbin('post'), data=d).from_cache) def test_post_data(self): # issue #2, raw payload self.s = CachedSession(CACHE_NAME, CACHE_BACKEND, allowable_methods=('GET', 'POST')) d1 = json.dumps({'param1': 'test1'}) d2 = json.dumps({'param1': 'test1', 'param2': 'test2'}) d3 = str('some unicode data') if is_py3: bin_data = bytes('some binary data', 'utf8') else: bin_data = bytes('some binary data') for d in (d1, d2, d3): self.assertEqual(self.post(d)['data'], d) r = self.s.post(httpbin('post'), data=d) self.assert_(hasattr(r, 'from_cache')) self.assertEqual(self.post(bin_data)['data'], bin_data.decode('utf8')) r = self.s.post(httpbin('post'), data=bin_data) self.assert_(hasattr(r, 'from_cache')) def test_get_params_as_argument(self): for _ in range(5): p = {'arg1': 'value1'} r = self.s.get(httpbin('get'), params=p) self.assert_(self.s.cache.has_url( httpbin('get?arg1=value1'))) def test_https_support(self): n = 10 delay = 1 url = 'https://httpbin.org/delay/%s?ar1=value1' % delay t = time.time() for _ in range(n): r = self.s.get(url, verify=False) self.assertLessEqual(time.time() - t, delay * n / 2) def test_from_cache_attribute(self): url = httpbin('get?q=1') self.assertFalse(self.s.get(url).from_cache) self.assertTrue(self.s.get(url).from_cache) self.s.cache.clear() self.assertFalse(self.s.get(url).from_cache) def test_gzip_response(self): url = httpbin('gzip') self.assertFalse(self.s.get(url).from_cache) self.assertTrue(self.s.get(url).from_cache)