def enable_http_cache(self):
try:
import cachecontrol
except ImportError:
raise ImportError(
"cachecontrol module is not installed."
" Install it like this from pip: $ pip install cachecontrol")
self.mount('https://', cachecontrol.CacheControlAdapter())
self.mount('http://', cachecontrol.CacheControlAdapter())
def _get_session(self):
if self._session is not None:
return self._session
session = requests.Session()
session.mount('https://', cachecontrol.CacheControlAdapter())
self._session = session
return self._session
def __init__(self, driver, connection_name, connection_config):
super(GithubConnection, self).__init__(driver, connection_name,
connection_config)
self._change_cache = {}
self._project_branch_cache = {}
self.projects = {}
self.git_ssh_key = self.connection_config.get('sshkey')
self.server = self.connection_config.get('server', 'github.com')
self.canonical_hostname = self.connection_config.get(
'canonical_hostname', self.server)
self.source = driver.getSource(self)
self.event_queue = queue.Queue()
if self.server == 'github.com':
self.base_url = GITHUB_BASE_URL
else:
self.base_url = 'https://%s/api/v3' % self.server
# ssl verification must default to true
verify_ssl = self.connection_config.get('verify_ssl', 'true')
self.verify_ssl = True
if verify_ssl.lower() == 'false':
self.verify_ssl = False
self._github = None
self.app_id = None
self.app_key = None
self.sched = None
self.installation_map = {}
self.installation_token_cache = {}
# NOTE(jamielennox): Better here would be to cache to memcache or file
# or something external - but zuul already sucks at restarting so in
# memory probably doesn't make this much worse.
# NOTE(tobiash): Unlike documented cachecontrol doesn't priorize
# the etag caching but doesn't even re-request until max-age was
# elapsed.
#
# Thus we need to add a custom caching heuristic which simply drops
# the cache-control header containing max-age. This way we force
# cachecontrol to only rely on the etag headers.
#
# http://cachecontrol.readthedocs.io/en/latest/etags.html
# http://cachecontrol.readthedocs.io/en/latest/custom_heuristics.html
class NoAgeHeuristic(BaseHeuristic):
def update_headers(self, response):
if 'cache-control' in response.headers:
del response.headers['cache-control']
self.cache_adapter = cachecontrol.CacheControlAdapter(
DictCache(), cache_etags=True, heuristic=NoAgeHeuristic())
# The regex is based on the connection host. We do not yet support
# cross-connection dependency gathering
self.depends_on_re = re.compile(
r"^Depends-On: https://%s/.+/.+/pull/[0-9]+$" % self.server,
re.MULTILINE | re.IGNORECASE)
def main():
cache_prefix = "https://"
file_cache = cachecontrol.caches.file_cache.FileCache(".advisory_cache")
cache_adapter = cachecontrol.CacheControlAdapter(cache=file_cache)
session = requests.Session()
session.mount(cache_prefix, cache_adapter)
conn = sqlite3.connect("advisories.db")
cur = conn.cursor()
cur.execute(
"CREATE TABLE IF NOT EXISTS advisories ("
"docid TEXT, "
"url TEXT, "
"title TEXT, "
"html TEXT, "
"automatic_classification TEXT"
")"
)
cur.execute("DELETE FROM advisories")
for result in full_details_gen(session):
print(
result.docid,
result.url,
result.title,
result.cwe_list,
result.automatic_classification
)
cur.execute(
"INSERT INTO advisories ("
"docid, "
"url, "
"title, "
"html, "
"automatic_classification"
") "
"VALUES(?, ?, ?, ?, ?)",
(
result.docid,
result.url,
result.title,
result.html,
result.automatic_classification.to_string(),
)
)
conn.commit()
def __init__(
self,
root,
apiname=None,
auth=None,
headers=None,
session=None,
cache=False,
curie=None,
):
self.root = utils.fix_scheme(root)
self.apiname = utils.namify(root) if apiname is None else apiname
self.uri = self.root
self.profile = None
self.title = None
self.type = 'application/hal+json'
self.default_curie = curie
self.curies = None
self.session = session or requests.Session()
if cache:
if isinstance(cache, cachecontrol.CacheControlAdapter):
cc = cache
else:
cc = cachecontrol.CacheControlAdapter()
self.session.mount('http://', cc)
self.session.mount('https://', cc)
self.session.auth = auth
self.session.headers.update(default_headers())
if headers:
self.session.headers.update(headers)
self.response = None
self.state = None
self.template_uri = None
self.template_args = None
self.parameters = None
self.templated = False
self._links = None
# This is the identity map shared by all descendents of this
# HALNavigator
self._id_map = WeakValueDictionary({self.root: self})
def new_session(args):
"""Returns a new Session with caching per supplied command line args."""
if args.debug:
logging.getLogger().setLevel(logging.DEBUG)
if args.debug_http:
logging.getLogger("urllib3").setLevel(logging.DEBUG)
logging.getLogger("cachecontrol").setLevel(logging.DEBUG)
session = requests.Session()
if args.cache_time:
logger.debug(f"Caching results for {args.cache_time}.")
args.cache_dir.mkdir(exist_ok=True)
adapter = cachecontrol.CacheControlAdapter(
cache=cachecontrol.caches.file_cache.FileCache(args.cache_dir),
heuristic=_CacheHeuristic(args.cache_time),
)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def __init__(self, credentialsFilename):
self.credentialsFilename = credentialsFilename
self.credentials = self._loadCredentialsFromFile(
self.credentialsFilename)
if not self.credentials:
raise Exception('credentials could not be loaded from file')
self.session = requests.session()
self.session.mount('https://', cachecontrol.CacheControlAdapter())
self.parser = HRootParser()
self.baseURL = self.credentials["baseURL"]
self.paths = {
"index": "/",
"login": "******",
"dashboard": "/admin",
"logout": "/logout"
}
self.experimentId = None
self.currentPage = ""
self.currentPageSource = ""
import threading
import warnings
from typing import Type
from pycryptoclients.exc import CCAPINoMethodException
from pycryptoclients.request import BaseCCRequest
from pycryptoclients.response import CCAPIResponseParser, CCAPIResponse
from pycryptoclients.utils import ENCODING
__all__ = ('APIMethod', 'BaseCCAPI', 'CCAPI', 'CCRPC')
try:
import cachecontrol
cache_adapter = cachecontrol.CacheControlAdapter()
except ImportError:
warnings.warn('Caching is not enabled. Install CacheControl for cache enabling', ImportWarning)
cache_adapter = None
class APIMethod(object):
def __init__(self, name: str, request: Type[BaseCCRequest], parser: Type[CCAPIResponseParser]):
self.name = name
self.request = request
self.parser = parser
ONE_MINUTE = 60.0
from future.moves.urllib.parse import urlencode
import requests
import gitlab
import cachecontrol
from requests.adapters import HTTPAdapter
from addons.gitlab.exceptions import NotFoundError, AuthError
from addons.gitlab.settings import DEFAULT_HOSTS
# Initialize caches
https_cache = cachecontrol.CacheControlAdapter()
default_adapter = HTTPAdapter()
class GitLabClient(object):
def __init__(self, external_account=None, access_token=None, host=None):
self.access_token = getattr(external_account, 'oauth_key',
None) or access_token
self.host = getattr(external_account, 'oauth_secret',
None) or host or DEFAULT_HOSTS[0]
if self.access_token:
self.gitlab = gitlab.Gitlab(self.host,
private_token=self.access_token)
else:
self.gitlab = gitlab.Gitlab(self.host)
def user(self, user=None):
"""Fetch a user or the authenticated user.
def __init__(self):
self.sess = requests.Session()
self.sess.mount("http://", cachecontrol.CacheControlAdapter())
self.sess.mount("https://", cachecontrol.CacheControlAdapter())
def _get_session(self):
session = requests.Session()
session.mount('https://', cachecontrol.CacheControlAdapter())
return session
clogger = logging.getLogger("cachecontrol")
clogger.addHandler(logging.StreamHandler())
clogger.setLevel(logging.DEBUG)
from pprint import pprint
class NoAgeHeuristic(BaseHeuristic):
def update_headers(self, response):
if "cache-control" in response.headers:
del response.headers["cache-control"]
cache_adapter = cachecontrol.CacheControlAdapter(DictCache(),
cache_etags=True,
heuristic=NoAgeHeuristic())
session = requests.Session()
session.mount("https://", cache_adapter)
def log_resp(resp):
return
print(f"{resp.status_code} {resp.request.method}")
for k, v in response.headers.items():
print(f"{k}: {v}")
for i in range(2):
