def test_redis_cache(self):
class FakeConnectionPool:
def __init__(self, name):
self.name = name
# our mock will be called here. Passing a connection_pool with a name
# makes sure that we can identify the different redis mocks
redis_mock_1 = redis.Redis(
connection_pool=FakeConnectionPool(name="test1")
)
redis_mock_2 = redis.Redis(
connection_pool=FakeConnectionPool(name="test2")
)
self.assertNotEqual(redis_mock_1, redis_mock_2)
def request_callback(request, uri, response_headers):
return [200, response_headers, json.dumps({"epoch": time.time()})]
httpretty.register_uri(
httpretty.GET, "https://now.httpbin.org", body=request_callback
)
with freeze_time("2012-01-14 12:00:01") as freezer:
session_1 = CachedSession(
redis_connection=redis_mock_1, fallback_cache_duration=500
)
session_2 = CachedSession(
redis_connection=redis_mock_2, fallback_cache_duration=1
)
resp_1 = session_1.get("https://now.httpbin.org")
resp_2 = session_2.get("https://now.httpbin.org")
self.assertNotEqual(resp_1.text, resp_2.text)
freezer.tick()
resp_3 = session_2.get("https://now.httpbin.org")
self.assertNotEqual(resp_2.text, resp_3.text)
session_3 = CachedSession(
redis_connection=redis_mock_1, fallback_cache_duration=1
)
resp_4 = session_3.get("https://now.httpbin.org")
self.assertEqual(resp_1.text, resp_4.text)
def test_default_heuristic(self):
def request_callback(request, uri, response_headers):
return [200, response_headers, json.dumps({"epoch": time.time()})]
httpretty.register_uri(
httpretty.GET, "https://now.httpbin.org", body=request_callback
)
session = CachedSession(file_cache_directory=file_cache_directory)
with freeze_time("2012-01-14 12:00:01") as freezer:
response_1 = session.get("https://now.httpbin.org")
freezer.tick()
freezer.tick()
response_2 = session.get("https://now.httpbin.org")
freezer.tick()
freezer.tick()
freezer.tick()
response_3 = session.get("https://now.httpbin.org")
self.assertEqual(response_1.text, response_2.text)
self.assertNotEqual(response_2.text, response_3.text)
def test_custom_heuristic(self):
def request_callback(request, uri, response_headers):
return [200, response_headers, json.dumps({"epoch": time.time()})]
httpretty.register_uri(
httpretty.GET, "https://now.httpbin.org", body=request_callback
)
session = CachedSession(
fallback_cache_duration=2,
file_cache_directory=file_cache_directory,
)
# with a 2s retention, and a 1.1s time between requests, 2 of the
# request should have the same epoch, where as the 3rd gets fresh data
# the first requests gets send at t=0
with freeze_time("2012-01-14 12:00:01") as freezer:
response_1 = session.get("https://now.httpbin.org")
freezer.tick()
response_2 = session.get("https://now.httpbin.org")
freezer.tick()
response_3 = session.get("https://now.httpbin.org")
self.assertEqual(response_1.text, response_2.text)
self.assertNotEqual(response_2.text, response_3.text)
def __init__(self,
base_url,
session=CachedSession(fallback_cache_duration=300)):
"""
@param base_url: The Discourse URL (e.g. https://discourse.example.com)
"""
self.base_url = base_url.rstrip("/")
self.session = session
def test_file_cache(self):
def request_callback(request, uri, response_headers):
return [200, response_headers, json.dumps({"epoch": time.time()})]
httpretty.register_uri(
httpretty.GET, "https://now.httpbin.org", body=request_callback
)
cache_dir_1 = ".test1"
cache_dir_2 = ".test2"
session_1 = CachedSession(
file_cache_directory=cache_dir_1, fallback_cache_duration=2000
)
session_2 = CachedSession(file_cache_directory=cache_dir_2)
resp_1 = session_1.get("https://now.httpbin.org")
self.assertEqual(os.path.isdir(cache_dir_1), True)
resp_2 = session_2.get("https://now.httpbin.org")
self.assertEqual(os.path.isdir(cache_dir_2), True)
self.assertNotEqual(resp_1.text, resp_2.text)
shutil.rmtree(cache_dir_2)
self.assertEqual(os.path.isdir(cache_dir_2), False)
resp_3 = session_2.get("https://now.httpbin.org")
self.assertEqual(os.path.isdir(cache_dir_2), True)
self.assertNotEqual(resp_2.text, resp_3.text)
session_3 = CachedSession(
file_cache_directory=cache_dir_1, fallback_cache_duration=2000
)
resp_4 = session_3.get("https://now.httpbin.org")
self.assertEqual(resp_1.text, resp_4.text)
shutil.rmtree(cache_dir_1)
shutil.rmtree(cache_dir_2)
def __init__(self, base_url, frontpage_id, session_class=CachedSession):
"""
@param base_url: The Discourse URL (e.g. https://discourse.example.com)
@param frontpage_id: The ID of the frontpage topic in Discourse.
This topic should also contain the navigation.
"""
self.base_url = base_url.rstrip("/")
self.frontpage_id = frontpage_id
self.session = CachedSession(expire_after=300)
def test_timeout_adapter(self):
session = CachedSession(
timeout=2, file_cache_directory=file_cache_directory
)
# this test can be inconsistent on multiple concurrent
# runs due to the use of time.sleep
with self.assertRaises(
(
requests.exceptions.ConnectTimeout,
requests.exceptions.ReadTimeout,
)
):
session.get("https://httpbin.org/delay/3")
resp = session.get("https://httpbin.org/delay/1")
self.assertIsNotNone(resp)
def test_cache_control_no_cache_overwrites_custom_heuristic(self):
def request_callback(request, uri, response_headers):
return [200, response_headers, json.dumps({"epoch": time.time()})]
httpretty.register_uri(
httpretty.GET,
"https://now.httpbin.org",
body=request_callback,
adding_headers={"Cache-Control": "no-cache"},
)
session = CachedSession(file_cache_directory=file_cache_directory)
# with no-cache set, no request should be cached,
# thus all bodies are different
response_1 = session.get("https://now.httpbin.org")
response_2 = session.get("https://now.httpbin.org")
response_3 = session.get("https://now.httpbin.org")
self.assertNotEqual(response_1.text, response_2.text)
self.assertNotEqual(response_2.text, response_3.text)
def get_search_results(
api_key,
query,
search_engine_id,
start=None,
num=None,
siteSearch=None,
session=CachedSession(fallback_cache_duration=600),
):
"""
Query the Google Custom Search API for search results
"""
response = session.get(
"https://www.googleapis.com/customsearch/v1",
params={
"key": api_key,
"cx": search_engine_id,
"q": query,
"start": start,
"num": num,
"siteSearch": siteSearch,
},
)
response.raise_for_status()
results = response.json()
if "items" in results:
# Move "items" to "entries" as "items" is a method name for dicts
results["entries"] = results.pop("items")
# Remove newlines from the snippet
for item in results["entries"]:
if "htmlSnippet" in item:
item["htmlSnippet"] = item["htmlSnippet"].replace("<br>\n", "")
return results
def test_cache_control_max_age_overwrites_custom_heuristic(self):
def request_callback(request, uri, response_headers):
return [200, response_headers, json.dumps({"epoch": time.time()})]
httpretty.register_uri(
httpretty.GET,
"https://now.httpbin.org",
body=request_callback,
adding_headers={"Cache-Control": "max-age=2"},
)
session = CachedSession(file_cache_directory=file_cache_directory)
with freeze_time("2012-01-14 12:00:01") as freezer:
response_1 = session.get("https://now.httpbin.org")
freezer.tick()
response_2 = session.get("https://now.httpbin.org")
freezer.tick()
response_3 = session.get("https://now.httpbin.org")
self.assertEqual(response_1.text, response_2.text)
self.assertNotEqual(response_2.text, response_3.text)
import os
from canonicalwebteam.http import CachedSession
API_URL = os.getenv("BLOG_API",
"https://admin.insights.ubuntu.com/wp-json/wp/v2")
api_session = CachedSession(fallback_cache_duration=3600)
tags = {}
def process_response(response):
response.raise_for_status()
return response.json()
def build_get_article_url(
tags=[],
per_page=12,
page=1,
tags_exclude=[],
exclude=[],
categories=[],
sticky="",
groups=[],
after="",
before="",
author="",
):
# Packages
from canonicalwebteam.http import CachedSession
# Constants
SEARCH_SESSION = CachedSession(fallback_cache_duration=600)
class NoAPIKeyError(Exception):
pass
def get_search_results(
api_key,
api_url,
search_custom_id,
query,
start,
num,
session=SEARCH_SESSION,
):
"""
Query the Google Custom Search API for search results
"""
if not api_key:
raise NoAPIKeyError("Unable to search: No API key provided")
results = session.get(
api_url,
params={
"key": api_key,
"cx": search_custom_id,
# Packages
from canonicalwebteam.http import CachedSession
# Constants
SEARCH_SESSION = CachedSession(expire_after=600)
class NoAPIKeyError(Exception):
pass
def get_search_results(
api_key,
api_url,
search_custom_id,
query,
start,
num,
session=SEARCH_SESSION,
):
"""
Query the Google Custom Search API for search results
"""
if not api_key:
raise NoAPIKeyError("Unable to search: No API key provided")
results = session.get(
api_url,
params={
"key": api_key,
"cx": search_custom_id,
from django.views.generic.base import TemplateView
from django_template_finder_view import TemplateFinder
from django.conf import settings
from django.shortcuts import render
from feedparser import parse
from canonicalwebteam.http import CachedSession
import traceback
try:
from urllib.parse import urlencode
except ImportError:
from urllib import urlencode
# Search service
if settings.SEARCH_API_KEY:
search_session = CachedSession(
expire_after=settings.SEARCH_CACHE_EXPIRY_SECONDS)
def _get_search_results(query, start, num):
"""
Query the Google Custom Search API for search results
"""
if not settings.SEARCH_API_KEY:
raise Exception('Unable to search: No API key provided')
results = search_session.get(settings.SEARCH_API_URL,
params={
'key': settings.SEARCH_API_KEY,
'cx': settings.CUSTOM_SEARCH_ID,
'q': query,
import base64
import json
import os
import re
import requests
from canonicalwebteam.http import CachedSession
from html import unescape
api_session = CachedSession(fallback_cache_duration=300,
file_cache_directory=".webcache")
base_url = "https://boards-api.greenhouse.io/v1/boards/Canonical/jobs"
def get_vacancies(department):
feed = api_session.get(f"{base_url}?content=true").json()
path_department = remove_hyphens(department)
vacancies = []
for job in feed["jobs"]:
feed_department = remove_hyphens(job["metadata"][2]["value"])
if path_department.lower() == "all":
vacancies.append({
"title": job["title"],
"content": unescape(job["content"]),
"url": job["absolute_url"],
"location": job["location"]["name"],
"id": job["id"],
"employment": job["metadata"][0]["value"],
"date": job["metadata"][1]["value"],
"department": job["metadata"][2]["value"],
from copy import copy
from django.views.generic.base import TemplateView
from django.conf import settings
from django.shortcuts import render
from feedparser import parse
from canonicalwebteam.http import CachedSession
import traceback
try:
from urllib.parse import urlencode
except ImportError:
from urllib import urlencode
# Search service
if settings.SEARCH_API_KEY:
search_session = CachedSession(
fallback_cache_duration=settings.SEARCH_CACHE_EXPIRY_SECONDS)
def _get_search_results(query, start, num, site=None):
"""
Query the Google Custom Search API for search results
"""
if not settings.SEARCH_API_KEY:
raise Exception("Unable to search: No API key provided")
results = search_session.get(
settings.SEARCH_API_URL,
params={
"key": settings.SEARCH_API_KEY,
"cx": settings.CUSTOM_SEARCH_ID,
import logging
from canonicalwebteam.http import CachedSession
# this part is temporarily included until
# https://github.com/canonical-webteam/get-feeds
# is updated for flask applications
requests_timeout = 10
expiry_seconds = 300
cached_request = CachedSession(fallback_cache_duration=expiry_seconds)
logger = logging.getLogger(__name__)
def get(url):
try:
response = cached_request.get(url, timeout=requests_timeout)
response.raise_for_status()
except Exception as request_error:
logger.warning("Attempt to get feed failed: {}".format(
str(request_error)))
return ""
return response
# Core
import re
# Third-party
import dateutil.parser
import humanize
from bs4 import BeautifulSoup
from requests.exceptions import HTTPError
from urllib.parse import urlparse
from canonicalwebteam.http import CachedSession
from jinja2 import Template
# Constants
DEFAULT_SESSION = CachedSession(expire_after=300, old_data_on_error=True)
class RedirectFoundError(HTTPError):
"""
If we encounter redirects from Discourse, we need to take action
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
url_parts = urlparse(self.response.headers["Location"])
self.redirect_path = re.sub("/t(/.*).json", r"\1", url_parts.path)
class NavigationParseError(Exception):
"""
Indicates a failure to extract the navigation from
the frontpage content
"""