def enable_http_cache(self): try: import cachecontrol except ImportError: raise ImportError( "cachecontrol module is not installed." " Install it like this from pip: $ pip install cachecontrol") self.mount('https://', cachecontrol.CacheControlAdapter()) self.mount('http://', cachecontrol.CacheControlAdapter())
def _get_session(self): if self._session is not None: return self._session session = requests.Session() session.mount('https://', cachecontrol.CacheControlAdapter()) self._session = session return self._session
def __init__(self, driver, connection_name, connection_config): super(GithubConnection, self).__init__(driver, connection_name, connection_config) self._change_cache = {} self._project_branch_cache = {} self.projects = {} self.git_ssh_key = self.connection_config.get('sshkey') self.server = self.connection_config.get('server', 'github.com') self.canonical_hostname = self.connection_config.get( 'canonical_hostname', self.server) self.source = driver.getSource(self) self.event_queue = queue.Queue() if self.server == 'github.com': self.base_url = GITHUB_BASE_URL else: self.base_url = 'https://%s/api/v3' % self.server # ssl verification must default to true verify_ssl = self.connection_config.get('verify_ssl', 'true') self.verify_ssl = True if verify_ssl.lower() == 'false': self.verify_ssl = False self._github = None self.app_id = None self.app_key = None self.sched = None self.installation_map = {} self.installation_token_cache = {} # NOTE(jamielennox): Better here would be to cache to memcache or file # or something external - but zuul already sucks at restarting so in # memory probably doesn't make this much worse. # NOTE(tobiash): Unlike documented cachecontrol doesn't priorize # the etag caching but doesn't even re-request until max-age was # elapsed. # # Thus we need to add a custom caching heuristic which simply drops # the cache-control header containing max-age. This way we force # cachecontrol to only rely on the etag headers. # # http://cachecontrol.readthedocs.io/en/latest/etags.html # http://cachecontrol.readthedocs.io/en/latest/custom_heuristics.html class NoAgeHeuristic(BaseHeuristic): def update_headers(self, response): if 'cache-control' in response.headers: del response.headers['cache-control'] self.cache_adapter = cachecontrol.CacheControlAdapter( DictCache(), cache_etags=True, heuristic=NoAgeHeuristic()) # The regex is based on the connection host. We do not yet support # cross-connection dependency gathering self.depends_on_re = re.compile( r"^Depends-On: https://%s/.+/.+/pull/[0-9]+$" % self.server, re.MULTILINE | re.IGNORECASE)
def main(): cache_prefix = "https://" file_cache = cachecontrol.caches.file_cache.FileCache(".advisory_cache") cache_adapter = cachecontrol.CacheControlAdapter(cache=file_cache) session = requests.Session() session.mount(cache_prefix, cache_adapter) conn = sqlite3.connect("advisories.db") cur = conn.cursor() cur.execute( "CREATE TABLE IF NOT EXISTS advisories (" "docid TEXT, " "url TEXT, " "title TEXT, " "html TEXT, " "automatic_classification TEXT" ")" ) cur.execute("DELETE FROM advisories") for result in full_details_gen(session): print( result.docid, result.url, result.title, result.cwe_list, result.automatic_classification ) cur.execute( "INSERT INTO advisories (" "docid, " "url, " "title, " "html, " "automatic_classification" ") " "VALUES(?, ?, ?, ?, ?)", ( result.docid, result.url, result.title, result.html, result.automatic_classification.to_string(), ) ) conn.commit()
def __init__( self, root, apiname=None, auth=None, headers=None, session=None, cache=False, curie=None, ): self.root = utils.fix_scheme(root) self.apiname = utils.namify(root) if apiname is None else apiname self.uri = self.root self.profile = None self.title = None self.type = 'application/hal+json' self.default_curie = curie self.curies = None self.session = session or requests.Session() if cache: if isinstance(cache, cachecontrol.CacheControlAdapter): cc = cache else: cc = cachecontrol.CacheControlAdapter() self.session.mount('http://', cc) self.session.mount('https://', cc) self.session.auth = auth self.session.headers.update(default_headers()) if headers: self.session.headers.update(headers) self.response = None self.state = None self.template_uri = None self.template_args = None self.parameters = None self.templated = False self._links = None # This is the identity map shared by all descendents of this # HALNavigator self._id_map = WeakValueDictionary({self.root: self})
def new_session(args): """Returns a new Session with caching per supplied command line args.""" if args.debug: logging.getLogger().setLevel(logging.DEBUG) if args.debug_http: logging.getLogger("urllib3").setLevel(logging.DEBUG) logging.getLogger("cachecontrol").setLevel(logging.DEBUG) session = requests.Session() if args.cache_time: logger.debug(f"Caching results for {args.cache_time}.") args.cache_dir.mkdir(exist_ok=True) adapter = cachecontrol.CacheControlAdapter( cache=cachecontrol.caches.file_cache.FileCache(args.cache_dir), heuristic=_CacheHeuristic(args.cache_time), ) session.mount("http://", adapter) session.mount("https://", adapter) return session
def __init__(self, credentialsFilename): self.credentialsFilename = credentialsFilename self.credentials = self._loadCredentialsFromFile( self.credentialsFilename) if not self.credentials: raise Exception('credentials could not be loaded from file') self.session = requests.session() self.session.mount('https://', cachecontrol.CacheControlAdapter()) self.parser = HRootParser() self.baseURL = self.credentials["baseURL"] self.paths = { "index": "/", "login": "******", "dashboard": "/admin", "logout": "/logout" } self.experimentId = None self.currentPage = "" self.currentPageSource = ""
import threading import warnings from typing import Type from pycryptoclients.exc import CCAPINoMethodException from pycryptoclients.request import BaseCCRequest from pycryptoclients.response import CCAPIResponseParser, CCAPIResponse from pycryptoclients.utils import ENCODING __all__ = ('APIMethod', 'BaseCCAPI', 'CCAPI', 'CCRPC') try: import cachecontrol cache_adapter = cachecontrol.CacheControlAdapter() except ImportError: warnings.warn('Caching is not enabled. Install CacheControl for cache enabling', ImportWarning) cache_adapter = None class APIMethod(object): def __init__(self, name: str, request: Type[BaseCCRequest], parser: Type[CCAPIResponseParser]): self.name = name self.request = request self.parser = parser ONE_MINUTE = 60.0
from future.moves.urllib.parse import urlencode import requests import gitlab import cachecontrol from requests.adapters import HTTPAdapter from addons.gitlab.exceptions import NotFoundError, AuthError from addons.gitlab.settings import DEFAULT_HOSTS # Initialize caches https_cache = cachecontrol.CacheControlAdapter() default_adapter = HTTPAdapter() class GitLabClient(object): def __init__(self, external_account=None, access_token=None, host=None): self.access_token = getattr(external_account, 'oauth_key', None) or access_token self.host = getattr(external_account, 'oauth_secret', None) or host or DEFAULT_HOSTS[0] if self.access_token: self.gitlab = gitlab.Gitlab(self.host, private_token=self.access_token) else: self.gitlab = gitlab.Gitlab(self.host) def user(self, user=None): """Fetch a user or the authenticated user.
def __init__(self): self.sess = requests.Session() self.sess.mount("http://", cachecontrol.CacheControlAdapter()) self.sess.mount("https://", cachecontrol.CacheControlAdapter())
def _get_session(self): session = requests.Session() session.mount('https://', cachecontrol.CacheControlAdapter()) return session
clogger = logging.getLogger("cachecontrol") clogger.addHandler(logging.StreamHandler()) clogger.setLevel(logging.DEBUG) from pprint import pprint class NoAgeHeuristic(BaseHeuristic): def update_headers(self, response): if "cache-control" in response.headers: del response.headers["cache-control"] cache_adapter = cachecontrol.CacheControlAdapter(DictCache(), cache_etags=True, heuristic=NoAgeHeuristic()) session = requests.Session() session.mount("https://", cache_adapter) def log_resp(resp): return print(f"{resp.status_code} {resp.request.method}") for k, v in response.headers.items(): print(f"{k}: {v}") for i in range(2):