def new_session(insecure=False, ca_file=None, total_retries=None): session = requests.Session() if total_retries is not None: http_adapter = adapters.HTTPAdapter(max_retries=retry.Retry( total=total_retries)) https_adapter = adapters.HTTPAdapter(max_retries=retry.Retry( total=total_retries)) session.mount('http://', http_adapter) session.mount('https://', https_adapter) session.verify = ca_file if ca_file else not insecure return session
def requests_with_retry(retries: int = 3) -> requests.Session: session = requests.Session() adapter = requests.adapters.HTTPAdapter(max_retries=retry.Retry( total=retries, backoff_factor=0.1, status_forcelist=(500, 502, 504))) session.mount("http://", adapter) session.mount("https://", adapter) return session
def __init__(self, base_url: str, *args, **kwargs): """ Creates a new HTTP session instance. Args: base_url: The base URL for this HTTP session *args: The optional positional arguments **kwargs: The optional keyword arguments """ super().__init__(*args, **kwargs) self._base_url = base_url # Custom headers self.headers.update({ 'User-Agent': 'geodatabr/{version} ({url})'.format(version=__meta__.__version__, url=__meta__.__url__), }) # Automatic retries for protocol in ('http://', 'https://'): self.mount( protocol, adapters.HTTPAdapter(max_retries=retry.Retry( total=HTTP_MAX_RETRIES, backoff_factor=HTTP_BACKOFF_FACTOR, status_forcelist=HTTP_RETRY_STATUSES)))
def _make_session(self, max_retries, retry_backoff, auth_url, endpoint): """Create requests session that retries on connection failures""" if max_retries is None: max_retries = constants.DEFAULT_RETRIES if retry_backoff is None: retry_backoff = constants.DEFAULT_RETRY_BACKOFF session = requests.Session() # this makes it retry on timeouts, connection errors, # and 503 Unavailable responses adapter = requests.adapters.HTTPAdapter( max_retries=requests_retry.Retry( total=max(0, max_retries), backoff_factor=max(0, retry_backoff), status_forcelist=set([503]), method_whitelist=set(), ) ) # Retries for authentication and API requests session.mount(auth_url, adapter) session.mount(endpoint, adapter) return session
def reconcile_rmq_vhost(): # pragma: no cover url = settings.RMQ_API_ENDPOINT vhost_name = settings.RMQ_VHOST user = settings.RMQ_USER userPass = user + ":" + settings.RMQ_PASSWORD authHeader = "Basic " + base64.b64encode(userPass.encode()).decode("utf-8") payload = json.dumps({ "vhosts": [{ "name": vhost_name }], "permissions": [{ "user": user, "vhost": vhost_name, "configure": ".*", "write": ".*", "read": ".*" }], "policies": [{ "vhost": vhost_name, "name": "sync-nodes-" + vhost_name, "pattern": ".*", "apply-to": "all", "definition": { "ha-sync-mode": "automatic", "ha-mode": "all" }, "priority": 0 }] }) headers = { 'authorization': authHeader, 'content-type': "application/json", } requests_session = requests.Session() parsed_url = urlparse(url) retry_adapter = requests.adapters.HTTPAdapter( max_retries=requests_retry.Retry( total=400, backoff_factor=0.5, method_whitelist=set(['POST']), )) requests_session.mount('{scheme}://'.format(scheme=parsed_url.scheme), retry_adapter) response = requests_session.request("POST", url, data=payload, headers=headers) if response.ok: print('Created vhost and permissions for: {vhost}'.format( vhost=vhost_name)) else: print( "Could not create vhost {vhost} on RabbitMQ management API endpoint {api_endpoint}: {reason}" .format(vhost=vhost_name, api_endpoint=url, reason=response.reason)) response.raise_for_status()
def _make_session(self): session = requests.Session() retry_conf = retry.Retry(total=self.tries, connect=self.tries, read=self.tries, backoff_factor=1, status_forcelist=[429]) retry_conf.BACKOFF_MAX = 8 adapter = requests.adapters.HTTPAdapter(max_retries=retry_conf) session.mount('http://', adapter) session.mount('https://', adapter) return session
def requests_context(): session = requests.Session() retry = urllib_retry.Retry(total=10, backoff_factor=1, status_forcelist=(500, 502, 503, 504)) adapter = requests_adapters.HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) return session
def _retrying_request_session(retries=10, backoff_factor=0.5): """Returns a requests session with retry logic.""" retry_config = retry.Retry(total=retries, connect=retries, read=retries, method_whitelist=['GET', 'POST'], backoff_factor=backoff_factor) adapter = adapters.HTTPAdapter(max_retries=retry_config) session = requests.Session() session.mount('http://', adapter) session.mount('https://', adapter) return session
class HTTPRequest: """ A requests wrapper preconfigured with best practice timeout and retry policies and per-thread session tracking. The timeout and retry policies are configured to turn on exponential backoff support provided by urllib3, and to make the client resistant to both network issues and intermittent server-side errors. The per-thread session tracking is to avoid many threads sharing the same session in multithreaded environments. """ retry_policy = retry.Retry(read=4, status=4, backoff_factor=0.1, status_forcelist=frozenset({500, 502, 503, 504})) timeout_policy = timeout.Timeout(connect=20, read=40) codes = requests.codes def __init__(self, max_redirects=1024, obey_retry_after=True): self.sessions = {} self.max_redirects = max_redirects self.obey_retry_after = obey_retry_after def __call__(self, *args, **kwargs): if get_ident() not in self.sessions: session = Session() session.max_redirects = self.max_redirects session.obey_retry_after = self.obey_retry_after adapter = HTTPAdapter(max_retries=self.retry_policy) session.mount('http://', adapter) session.mount('https://', adapter) self.sessions[get_ident()] = session return self.sessions[get_ident()].request(*args, timeout=self.timeout_policy, **kwargs) def head(self, url, *args, **kwargs): return self(url=url, method="HEAD", *args, **kwargs) def get(self, url, *args, **kwargs): return self(url=url, method="GET", *args, **kwargs) def put(self, url, *args, **kwargs): return self(url=url, method="PUT", *args, **kwargs) def post(self, url, *args, **kwargs): return self(url=url, method="POST", *args, **kwargs) def delete(self, url, *args, **kwargs): return self(url=url, method="DELETE", *args, **kwargs) def patch(self, url, *args, **kwargs): return self(url=url, method="PATCH", *args, **kwargs) def options(self, url, *args, **kwargs): return self(url=url, method="OPTIONS", *args, **kwargs)
def __init__(self): self.bugzilla = hotness.bz.Bugzilla(consumer=self, config=hotness_config["bugzilla"]) self.buildsys = hotness.buildsys.Koji(consumer=self, config=hotness_config["koji"]) self.pdc_url = hotness_config["pdc_url"] self.dist_git_url = hotness_config["dist_git_url"] self.anitya_url = hotness_config["anitya"]["url"] # Also, set up our global cache object. _log.info("Configuring cache.") with hotness.cache.cache_lock: if not hotness.cache.cache.is_configured: hotness.cache.cache.configure(**hotness_config["cache"]) self.mdapi_url = hotness_config["mdapi_url"] _log.info("Using hotness.mdapi_url=%r" % self.mdapi_url) self.repoid = hotness_config["repoid"] _log.info("Using hotness.repoid=%r" % self.repoid) self.distro = hotness_config["distro"] _log.info("Using hotness.distro=%r" % self.distro) # Retrieve the requests configuration; by default requests time out # after 15 seconds and are retried up to 3 times. self.requests_session = requests.Session() self.timeout = ( hotness_config["connect_timeout"], hotness_config["read_timeout"], ) retries = hotness_config["requests_retries"] retry_conf = retry.Retry(total=retries, connect=retries, read=retries, backoff_factor=1) retry_conf.BACKOFF_MAX = 5 self.requests_session.mount( "http://", requests.adapters.HTTPAdapter(max_retries=retry_conf)) self.requests_session.mount( "https://", requests.adapters.HTTPAdapter(max_retries=retry_conf)) _log.info("Requests timeouts are {}s (connect) and {}s (read)" " with {} retries".format(self.timeout[0], self.timeout[1], retries)) # Build a little store where we'll keep track of what koji scratch # builds we have kicked off. We'll look later for messages indicating # that they have completed. self.scratch_builds = {} _log.info("That new hotness ticket filer is all initialized")
def requests_with_retry(retries: int = 3) -> requests.Session: """Creates a `requests` object. Request will be attempted `retries` times. Will retry on 500, 502 and 504 status codes. >>> response = requests_with_retry(3).get("http://someurl.com") """ session = requests.Session() adapter = requests.adapters.HTTPAdapter(max_retries=retry.Retry( total=retries, backoff_factor=0.1, status_forcelist=(500, 502, 504), ), ) session.mount("http://", adapter) session.mount("https://", adapter) return session
def _get_session(self) -> requests.Session: """ Initialize and return a session allows make retry when some errors will raised. """ session = requests.Session() retry = rq_retry.Retry( total=self.retries_number, read=self.retries_number, connect=self.retries_number, backoff_factor=self.backoff_factor, status_forcelist=self.status_forcelist, ) adapter = rq_adapt.HTTPAdapter(max_retries=retry) session.mount("https://", adapter) return session
def __init__(self, config, event_listener=None, tries=DEFAULT_TRIES, session=None): """ :param config: downloader configuration :type config: nectar.config.DownloaderConfig :param event_listener: event listener providing life-cycly callbacks :type event_listener: nectar.listener.DownloadEventListener :param tries: total number of requests made to the remote server, including first unsuccessful one :type tries: int :param session: The requests Session to use when downloaded. If one is not provided, one will be created and used for the lifetime of this downloader. :type session: requests.Session """ super(HTTPThreadedDownloader, self).__init__(config, event_listener) # throttling support self._bytes_lock = threading.RLock() self._bytes_this_second = 0 self._time_bytes_this_second_was_cleared = datetime.datetime.now() # thread-safety when firing events self._event_lock = threading.RLock() # default tries to fetch item self.tries = tries # set of locations that produced a connection error self.failed_netlocs = set([]) self.session = session or build_session(config) # Configure an adapter to retry failed requests. See urllib3's documentation # for details on each argument. retry_conf = retry.Retry(total=tries, connect=tries, read=tries, backoff_factor=1) retry_conf.BACKOFF_MAX = 8 adapter = requests.adapters.HTTPAdapter(max_retries=retry_conf) self.session.mount('http://', adapter) self.session.mount('https://', adapter)
def get_http(): """Get the http object.""" ensure_dir(CLUSTERFUZZ_TESTCASES_DIR) http = requests_cache.CachedSession(cache_name=os.path.join( CLUSTERFUZZ_TESTCASES_DIR, 'http_cache'), backend='sqlite', allowable_methods=('GET', 'POST'), allowable_codes=[200], expire_after=HTTP_CACHE_TTL) http.mount( 'https://', adapters.HTTPAdapter( # backoff_factor is 0.5. Therefore, the max wait time is 16s. retry.Retry(total=5, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504]))) return http
def requests_retry_session( retries=3, backoff_factor=0.3, status_forcelist=(500, 502, 504), session=None, ): session = session or requests.Session() max_retries = retry.Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, ) adapter = adapters.HTTPAdapter(max_retries=max_retries) session.mount('http://', adapter) session.mount('https://', adapter) return session
def url_helper(sess=None, retries=10, backoff_factor=0.3, status_forcelist=(500, 502, 504)): """url_helper :param sess: ``requests.Session`` object like .. code-block:: python s = requests.Session() s.auth = ('user', 'pass') s.headers.update({'x-test': 'true'}) response = url_helper(sesssion=s).get( 'https://www.peterbe.com' ) :param retries: number of retries default is ``3`` :param backoff_factor: seconds per attempt default is ``0.3`` :param status_forcelist: optional tuple list of retry error HTTP status codes default is ``500, 502, 504`` """ session = sess or requests.Session() retry = requests_retry.Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, ) adapter = adapters.HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) return session
class GSClient: base_url = "https://www.googleapis.com/storage/v1/" presigned_url_base = "https://storage.googleapis.com/" scope = "https://www.googleapis.com/auth/cloud-platform" instance_metadata_url = "http://metadata.google.internal/computeMetadata/v1/" svc_acct_token_url = instance_metadata_url + "instance/service-accounts/default/token" project_id_metadata_url = instance_metadata_url + "project/project-id" suppress_paging_warning = False retry_policy = retry.Retry(connect=5, read=5, status_forcelist=frozenset({500, 502, 503, 504}), backoff_factor=1) timeout = 20 def __init__(self, config=None, **session_kwargs): if config is None: config = tweak.Config(__name__, save_on_exit=False) self.config = config self._service_jwt = None self._oauth2_token = None self._sessions = {} self._session_kwargs = session_kwargs def get_session(self): thread_id = get_ident() if thread_id not in self._sessions: session = requests.Session(**self._session_kwargs) session.headers.update({"User-Agent": self.__class__.__name__}) token = self.get_oauth2_token() if token is not None: session.headers.update({"Authorization": "Bearer " + token}) adapter = HTTPAdapter(max_retries=self.retry_policy) session.mount('http://', adapter) session.mount('https://', adapter) self._sessions[thread_id] = session return self._sessions[thread_id] def get_oauth2_token(self): # TODO: invalidate and refetch before expiration if self._oauth2_token is None: try: service_jwt = self.get_service_jwt() params = dict( grant_type="urn:ietf:params:oauth:grant-type:jwt-bearer", assertion=service_jwt) res = requests.post( "https://www.googleapis.com/oauth2/v4/token", data=params) except NoServiceCredentials: try: res = requests.get(self.svc_acct_token_url, headers={"Metadata-Flavor": "Google"}) except Exception: logger.warn( 'API credentials not configured. Sending unsigned requests.' ) logger.warn( 'To set credentials, run "gs configure" or set GOOGLE_APPLICATION_CREDENTIALS.' ) return None res.raise_for_status() self._oauth2_token = res.json()["access_token"] return self._oauth2_token def get_service_jwt(self): if self._service_jwt is None: if "service_credentials" not in self.config: if "GOOGLE_APPLICATION_CREDENTIALS" in os.environ: logger.info("Using GOOGLE_APPLICATION_CREDENTIALS file %s", os.environ["GOOGLE_APPLICATION_CREDENTIALS"]) with open(os.environ["GOOGLE_APPLICATION_CREDENTIALS"] ) as fh: self.config.service_credentials = json.load(fh) else: raise NoServiceCredentials() payload = { 'iss': self.config.service_credentials["client_email"], 'sub': self.config.service_credentials["client_email"], 'scope': self.scope, 'aud': "https://www.googleapis.com/oauth2/v4/token", 'iat': datetime.datetime.utcnow(), 'exp': datetime.datetime.utcnow() + datetime.timedelta(minutes=60) } additional_headers = { 'kid': self.config.service_credentials["private_key_id"] } import jwt self._service_jwt = jwt.encode( payload, self.config.service_credentials["private_key"], headers=additional_headers, algorithm='RS256').decode() return self._service_jwt def request(self, method, resource, **kwargs): url = self.base_url + resource res = self.get_session().request(method=method, url=url, timeout=self.timeout, **kwargs) res.raise_for_status() return res if kwargs.get( "stream") is True or method == "delete" else res.json() def get(self, resource, **kwargs): return self.request(method="get", resource=resource, **kwargs) def post(self, resource, **kwargs): return self.request(method="post", resource=resource, **kwargs) def patch(self, resource, **kwargs): return self.request(method="patch", resource=resource, **kwargs) def put(self, resource, **kwargs): return self.request(method="put", resource=resource, **kwargs) def delete(self, resource, **kwargs): return self.request(method="delete", resource=resource, **kwargs) def get_project(self): if "GOOGLE_CLOUD_PROJECT" in os.environ: return os.environ["GOOGLE_CLOUD_PROJECT"] self.get_session( ) # Ensures any available project-specific credentials are loaded. if "service_credentials" in self.config: return self.config.service_credentials["project_id"] res = requests.get(self.project_id_metadata_url, headers={"Metadata-Flavor": "Google"}) res.raise_for_status() return res.content.decode() def list(self, resource, include_prefixes=True, **kwargs): while True: page = self.request(method="get", resource=resource, **kwargs) items = [dict(name=i) for i in page.get("prefixes", []) ] if include_prefixes else [] items.extend(page.get("items", [])) for item in items: yield item if "maxResults" in kwargs["params"]: kwargs["params"]["maxResults"] -= 1 if kwargs["params"]["maxResults"] == 0: return if "nextPageToken" in page: if not self.suppress_paging_warning: logger.warn( "Large number of results returned. Listing may take a while. " "You can limit the object count using the --max-results option." ) self.suppress_paging_warning = True kwargs["params"]["pageToken"] = page["nextPageToken"] else: break def get_presigned_url(self, bucket, key, expires_at, method="GET", headers=None, content_type=None, md5_b64=""): from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization, hashes from cryptography.hazmat.primitives.asymmetric import padding string_to_sign = "\n".join( [method, md5_b64, content_type or "", str(int(expires_at))]) for header, value in (headers.items() if headers else {}): string_to_sign += "\n" + header + ":" + value string_to_sign += "\n/" + bucket + "/" + key private_key_bytes = self.config.service_credentials[ "private_key"].encode() private_key = serialization.load_pem_private_key( private_key_bytes, password=None, backend=default_backend()) signature = private_key.sign(string_to_sign.encode(), padding.PKCS1v15(), hashes.SHA256()) qs = dict( GoogleAccessId=self.config.service_credentials["client_email"], Expires=str(int(expires_at)), Signature=base64.b64encode(signature).decode()) return self.presigned_url_base + bucket + "/" + key + "?" + requests.compat.urlencode( qs)
def __init__(self, hub): # If we're in development mode, rewrite some of our topics so that # local playback with fedmsg-dg-replay works as expected. if hub.config["environment"] == "dev": # Keep the original set, but append a duplicate set for local work prefix, env = hub.config["topic_prefix"], hub.config["environment"] self.topic = self.topic + [ ".".join([prefix, env] + topic.split(".")[3:]) for topic in self.topic ] super(BugzillaTicketFiler, self).__init__(hub) if not self._initialized: return # This is just convenient. self.config = self.hub.config # First, initialize fedmsg and bugzilla in this thread's context. hostname = socket.gethostname().split(".", 1)[0] if not getattr(getattr(fedmsg, "__local", None), "__context", None): fedmsg.init(name="hotness.%s" % hostname) fedmsg.meta.make_processors(**self.hub.config) self.bugzilla = hotness.bz.Bugzilla( consumer=self, config=self.config["hotness.bugzilla"] ) self.buildsys = hotness.buildsys.Koji( consumer=self, config=self.config["hotness.koji"] ) default = "https://pagure.io/releng/fedora-scm-requests" self.repo_url = self.config.get("hotness.repo_url", default) default = "https://pdc.fedoraproject.org" self.pdc_url = self.config.get("hotness.pdc_url", default) default = "https://src.fedoraproject.org" self.dist_git_url = self.config.get("hotness.dist_git_url", default) anitya_config = self.config.get("hotness.anitya", {}) default = "https://release-monitoring.org" self.anitya_url = anitya_config.get("url", default) self.anitya_username = anitya_config.get("username", default) self.anitya_password = anitya_config.get("password", default) # Also, set up our global cache object. _log.info("Configuring cache.") with hotness.cache.cache_lock: if not hotness.cache.cache.is_configured: hotness.cache.cache.configure(**self.config['hotness.cache']) self.mdapi_url = self.config.get("hotness.mdapi_url") _log.info("Using hotness.mdapi_url=%r" % self.mdapi_url) self.repoid = self.config.get("hotness.repoid", "rawhide") _log.info("Using hotness.repoid=%r" % self.repoid) self.distro = self.config.get("hotness.distro", "Fedora") _log.info("Using hotness.distro=%r" % self.distro) # Retrieve the requests configuration; by default requests time out # after 15 seconds and are retried up to 3 times. self.requests_session = requests.Session() self.timeout = ( self.config.get("hotness.connect_timeout", 15), self.config.get("hotness.read_timeout", 15), ) retries = self.config.get("hotness.requests_retries", 3) retry_conf = retry.Retry( total=retries, connect=retries, read=retries, backoff_factor=1 ) retry_conf.BACKOFF_MAX = 5 self.requests_session.mount( "http://", requests.adapters.HTTPAdapter(max_retries=retry_conf) ) self.requests_session.mount( "https://", requests.adapters.HTTPAdapter(max_retries=retry_conf) ) _log.info( "Requests timeouts are {}s (connect) and {}s (read)" " with {} retries".format(self.timeout[0], self.timeout[1], retries) ) # Build a little store where we'll keep track of what koji scratch # builds we have kicked off. We'll look later for messages indicating # that they have completed. self.scratch_builds = {} _log.info("That new hotness ticket filer is all initialized")
def __init__(self): # This is just convenient. self.config = conf["consumer_config"] self.bugzilla = hotness.bz.Bugzilla(consumer=self, config=self.config["bugzilla"]) self.buildsys = hotness.buildsys.Koji(consumer=self, config=self.config["koji"]) default = "https://pdc.fedoraproject.org" self.pdc_url = self.config.get("pdc_url", default) default = "https://src.fedoraproject.org" self.dist_git_url = self.config.get("dist_git_url", default) anitya_config = self.config.get("anitya", {}) default = "https://release-monitoring.org" self.anitya_url = anitya_config.get("url", default) self.anitya_username = anitya_config.get("username", default) self.anitya_password = anitya_config.get("password", default) # Also, set up our global cache object. _log.info("Configuring cache.") with hotness.cache.cache_lock: if not hotness.cache.cache.is_configured: hotness.cache.cache.configure(**self.config["cache"]) self.mdapi_url = self.config.get("mdapi_url") _log.info("Using hotness.mdapi_url=%r" % self.mdapi_url) self.repoid = self.config.get("repoid", "rawhide") _log.info("Using hotness.repoid=%r" % self.repoid) self.distro = self.config.get("distro", "Fedora") _log.info("Using hotness.distro=%r" % self.distro) # Retrieve the requests configuration; by default requests time out # after 15 seconds and are retried up to 3 times. self.requests_session = requests.Session() self.timeout = ( self.config.get("connect_timeout", 15), self.config.get("read_timeout", 15), ) retries = self.config.get("hotness.requests_retries", 3) retry_conf = retry.Retry(total=retries, connect=retries, read=retries, backoff_factor=1) retry_conf.BACKOFF_MAX = 5 self.requests_session.mount( "http://", requests.adapters.HTTPAdapter(max_retries=retry_conf)) self.requests_session.mount( "https://", requests.adapters.HTTPAdapter(max_retries=retry_conf)) _log.info("Requests timeouts are {}s (connect) and {}s (read)" " with {} retries".format(self.timeout[0], self.timeout[1], retries)) # Build a little store where we'll keep track of what koji scratch # builds we have kicked off. We'll look later for messages indicating # that they have completed. self.scratch_builds = {} _log.info("That new hotness ticket filer is all initialized")
TESTCASE_CACHE = LRUCacheDict(max_size=1000, expiration=172800) # The number of seconds to sleep after each test run to avoid DDOS. SLEEP_TIME = 30 Testcase = collections.namedtuple('Testcase', ['id', 'job_type']) # Configuring backoff retrying because sending a request to ClusterFuzz # might fail during a deployment. http = requests.Session() http.mount( 'https://', adapters.HTTPAdapter( # backoff_factor is 0.5. Therefore, the max wait time is 16s. retry.Retry(total=5, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504]))) def post(*args, **kwargs): # pragma: no cover """Make a post request. This method is needed for mocking.""" return http.post(*args, **kwargs) def load_sanity_check_testcase_ids(): """Return a list of all testcases to try.""" with open(SANITY_CHECKS) as stream: return yaml.load(stream)['testcase_ids'] def build_command(args):
DEFAULT_CONNECTION_TIMEOUT = 1 DEFAULT_READ_TIMEOUT = 3 DEFAULT_CONNECTION_RETRIES = 1 DEFAULT_READ_RETRIES = 1 DEFAULT_TOTAL_RETRIES = 1 DEFAULT_STATUS_CODES_TO_RETRY_ON = range(500, 600) Timeout = collections.namedtuple('Timeout', ['connection_timeout', 'read_timeout']) DEFAULT_TIMEOUT = Timeout(connection_timeout=DEFAULT_CONNECTION_TIMEOUT, read_timeout=DEFAULT_READ_TIMEOUT) DEFAULT_RETRY = retry.Retry( total=DEFAULT_TOTAL_RETRIES, connect=DEFAULT_CONNECTION_RETRIES, read=DEFAULT_READ_RETRIES, status_forcelist=DEFAULT_STATUS_CODES_TO_RETRY_ON, ) class ServiceCaller: """ A class for calling :mod:`services <apiron.service.base>` """ @staticmethod def get_adapted_session(adapter): """ Mounts an adapter capable of communication over HTTP or HTTPS to the supplied session. :param adapter: A :class:`requests.adapters.HTTPAdapter` instance
This module provides utilities for making HTTP calls using the requests library. """ from google.auth import transport import requests from requests.packages.urllib3.util import retry # pylint: disable=import-error _ANY_METHOD = None # Default retry configuration: Retries once on low-level connection and socket read errors. # Retries up to 4 times on HTTP 500 and 503 errors, with exponential backoff. Returns the # last response upon exhausting all retries. DEFAULT_RETRY_CONFIG = retry.Retry(connect=1, read=1, status_forcelist=[500, 503], method_whitelist=_ANY_METHOD, raise_on_status=False, backoff_factor=0.5) class HttpClient: """Base HTTP client used to make HTTP calls. HttpClient maintains an HTTP session, and handles request authentication and retries if necessary. """ def __init__(self, credential=None, session=None, base_url='', headers=None,
"""Internal HTTP client module. This module provides utilities for making HTTP calls using the requests library. """ from google.auth import transport import requests from requests.packages.urllib3.util import retry # pylint: disable=import-error # Default retry configuration: Retries once on low-level connection and socket read errors. # Retries up to 4 times on HTTP 500 and 503 errors, with exponential backoff. Returns the # last response upon exhausting all retries. DEFAULT_RETRY_CONFIG = retry.Retry( connect=1, read=1, status=4, status_forcelist=[500, 503], raise_on_status=False, backoff_factor=0.5) class HttpClient(object): """Base HTTP client used to make HTTP calls. HttpClient maintains an HTTP session, and handles request authentication and retries if necessary. """ def __init__( self, credential=None, session=None, base_url='', headers=None, retries=DEFAULT_RETRY_CONFIG): """Creates a new HttpClient instance from the provided arguments.
def __init__(self, hub): # If we're in development mode, rewrite some of our topics so that # local playback with fedmsg-dg-replay works as expected. if hub.config['environment'] == 'dev': # Keep the original set, but append a duplicate set for local work prefix, env = hub.config['topic_prefix'], hub.config['environment'] self.topic = self.topic + [ '.'.join([prefix, env] + topic.split('.')[3:]) for topic in self.topic ] super(BugzillaTicketFiler, self).__init__(hub) if not self._initialized: return # This is just convenient. self.config = self.hub.config # First, initialize fedmsg and bugzilla in this thread's context. hostname = socket.gethostname().split('.', 1)[0] fedmsg.init(name='hotness.%s' % hostname) fedmsg.meta.make_processors(**self.hub.config) self.bugzilla = hotness.bz.Bugzilla( consumer=self, config=self.config['hotness.bugzilla']) self.buildsys = hotness.buildsys.Koji( consumer=self, config=self.config['hotness.koji']) default = 'https://admin.fedoraproject.org/pkgdb/api' self.pkgdb_url = self.config.get('hotness.pkgdb_url', default) anitya_config = self.config.get('hotness.anitya', {}) default = 'https://release-monitoring.org' self.anitya_url = anitya_config.get('url', default) self.anitya_username = anitya_config.get('username', default) self.anitya_password = anitya_config.get('password', default) # Also, set up our global cache object. _log.info("Configuring cache.") with hotness.cache.cache_lock: if not hasattr(hotness.cache.cache, 'backend'): hotness.cache.cache.configure(**self.config['hotness.cache']) self.mdapi_url = self.config.get("hotness.mdapi_url") _log.info("Using hotness.mdapi_url=%r" % self.mdapi_url) self.repoid = self.config.get('hotness.repoid', 'rawhide') _log.info("Using hotness.repoid=%r" % self.repoid) self.distro = self.config.get('hotness.distro', 'Fedora') _log.info("Using hotness.distro=%r" % self.distro) # Retrieve the requests configuration; by default requests time out # after 15 seconds and are retried up to 3 times. self.requests_session = requests.Session() self.timeout = ( self.config.get('hotness.connect_timeout', 15), self.config.get('hotness.read_timeout', 15), ) retries = self.config.get('hotness.requests_retries', 3) retry_conf = retry.Retry(total=retries, connect=retries, read=retries, backoff_factor=1) retry_conf.BACKOFF_MAX = 5 self.requests_session.mount( 'http://', requests.adapters.HTTPAdapter(max_retries=retry_conf)) self.requests_session.mount( 'https://', requests.adapters.HTTPAdapter(max_retries=retry_conf)) _log.info('Requests timeouts are {}s (connect) and {}s (read)' ' with {} retries'.format(self.timeout[0], self.timeout[1], retries)) # Build a little store where we'll keep track of what koji scratch # builds we have kicked off. We'll look later for messages indicating # that they have completed. self.scratch_builds = {} _log.info("That new hotness ticket filer is all initialized")