def requests_session(): # Setup a session with a retry policy session = requests.Session() adapter = HTTPAdapter( max_retries=Retry( total=_UPLOAD_MAX_TRIES, backoff_factor=_UPLOAD_RETRY_BACKOFF, status_forcelist=[500, 502, 503, 504, 104], ) ) # Override longest-match adapter for http and https session.mount( "http://", adapter, ) session.mount( "https://", adapter, ) yield session
def __init__(self, host, user, password): retry_strategy = Retry( total=10, # status_forcelist=[429, 500, 502, 503, 504], method_whitelist=["HEAD", "GET", "OPTIONS"], backoff_factor=1 ) adapter = HTTPAdapter(max_retries=retry_strategy) self.http = requests.Session() self.http.mount("https://", adapter) self.http.mount("http://", adapter) self._auth = HTTPBasicAuth(user, password) self.http.auth = self._auth if host.startswith("http"): self._conn_string = host else: self._conn_string = "http://%s" % host
def __init__(self): self.oauth_url = "https://api.real-debrid.com/oauth/v2/" self.device_code_url = "device/code?{}" self.device_credentials_url = "device/credentials?{}" self.token_url = "token" self.device_code = "" self.oauth_timeout = 0 self.oauth_time_step = 0 self.base_url = "https://api.real-debrid.com/rest/1.0/" self.cache_check_results = {} self.progress_dialog = xbmcgui.DialogProgress() self.session = requests.Session() retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) self.session.mount("https://", HTTPAdapter(max_retries=retries, pool_maxsize=100)) self._load_settings()
def requests_retry_session( retries=3, backoff_factor=0.3, status_forcelist=(500, 502, 504), session=None, ): session = session or requests.Session() retry = Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, ) adapter = HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) return session
def get_http_session(retries=3, backoff_factor=0.3, status_forcelist=(500, 502, 504)) -> requests.Session: """ Taken from: https://www.peterbe.com/plog/best-practice-with-retries-with-requests """ session = requests.Session() retry = Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, ) adapter = HTTPAdapter(max_retries=retry) session.mount("http://", adapter) session.mount("https://", adapter) return session
def setup_configuration(api_key, host, username="", password="", verify_ssl=True): host = host + "/v1" global config config.api_key = api_key config.host = host config.username = username config.password = password config.verify_ssl = verify_ssl config.retries = Retry( total=10, backoff_factor=0.25, status_forcelist=[503], method_whitelist=False, raise_on_status=False, ) # Set logged in at this point global logged_in logged_in = True
def __init__( self, jira_url, auth, params=None, webbrowser_auth=False, jsessionid_for_testing=None, ): retries = 3 backoff_factor = 0.3 status_forcelist = (500, 502, 504) retry_session = None self.jira_url = jira_url self.insight_api_url = f"{jira_url}/rest/insight/1.0" self.auth = auth # Configure retry session self.retry_session = retry_session or requests.Session() retry = Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, ) adapter = HTTPAdapter(max_retries=retry) self.retry_session.mount("http://", adapter) self.retry_session.mount("https://", adapter) self.retry_session.auth = self.auth self.retry_session.params = params # TODO: Find out, why the lib does not extract session cookies if webbrowser_auth: from urllib.parse import urlparse url = urlparse(self.jira_url) import browser_cookie3 cookies = browser_cookie3.firefox(domain_name=url.netloc) self.retry_session.cookies = cookies if jsessionid_for_testing: self.retry_session.cookies = requests.cookies.cookiejar_from_dict( {"JSESSIONID": jsessionid_for_testing} )
def get_retry_policy(num_retry=3): """ :return: Returns the msrest or requests REST client retry policy. :rtype: urllib3.Retry """ status_forcelist = [413, 429, 500, 502, 503, 504] backoff_factor = 0.4 retry_policy = Retry( total=num_retry, read=num_retry, connect=num_retry, backoff_factor=backoff_factor, status_forcelist=status_forcelist, # By default this is True. We set it to false to get the full error trace, including url and # status code of the last retry. Otherwise, the error message is 'too many 500 error responses', # which is not useful. raise_on_status=False, ) return retry_policy
def requests_retry_session( # Thank you Peter Becom # https://www.peterbe.com/plog/best-practice-with-retries-with-requests retries=3, backoff_factor=5, status_forcelist=(500, 502, 504), session=None): session = session or requests.Session() retry = Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, ) adapter = HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) return session
def doi2bib(doi): retry_strategy = Retry(total=3, status_forcelist=[429, 500, 502, 503, 504], method_whitelist=["HEAD", "GET", "OPTIONS"]) adapter = HTTPAdapter(max_retries=retry_strategy) http = requests.Session() http.mount("https://", adapter) http.mount("http://", adapter) url = f"http://dx.doi.org/{doi}" headers = {'Accept': 'application/x-bibtex'} logging.info(f"Retrieving bibtex from {url}") r = http.get(url, headers=headers) # r = requests.get(url, headers=headers) if r.status_code is not 200: logging.info(r) raise RequestException("Failed to retrieve BIB for a DOI.") return r.text
def ensure_session(self, establish_retries=0): try: if establish_retries > 5: # maximum retries to login (get uid) # connecting retires specified below return False if self.get_uid(self.username): return True else: self.session = requests.Session() self.session.mount( 'https://', HTTPAdapter( max_retries=Retry(total=5, backoff_factor=0.1))) return self.ensure_session(establish_retries + 1) except: traceback.print_exc() return False
def _requests_retry_session( # A function for basic retrying of an url when it isn't accessible immediately. retries=8, backoff_factor=0.01, status_forcelist=(500, 502, 504), session=None, ): session = session or requests.Session() retry = Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, ) adapter = HTTPAdapter(max_retries=retry) session.mount("http://", adapter) session.mount("https://", adapter) return session
def _http_request(method: str, endpoint: str, handle_timeout: bool, **kwargs) -> Optional[JT]: url = build_url(endpoint) parsed_url = parse_url(url) pm_args = { "num_pools": constants.HTTP_POOL_MANAGER_COUNT, "host": parsed_url.host, "port": parsed_url.port, "retries": Retry(connect=constants.HTTP_REQUEST_RETRIES_COUNT, read=constants.HTTP_REQUEST_RETRIES_COUNT, redirect=constants.HTTP_REQUEST_RETRIES_COUNT, backoff_factor=constants.HTTP_REQUEST_BACKOFF_FACTOR, method_whitelist=METHODS_WHITELIST), "ssl_context": _ssl_context, } if _ssl_context is not None and url.startswith("https"): pm_args["assert_hostname"] = False http_pool_manager: PoolManager = PoolManager(**pm_args) try: logger.trace("HTTP {0} to {1}", method, url) response = http_pool_manager.request( method=method, url=parsed_url.url, timeout=constants.HTTP_REQUEST_TIMEOUT, **kwargs) raise_for_status(response) except MaxRetryError as e: logger.info("{} to {} failed due to: {}.", method, url, e) return None except TimeoutError: if handle_timeout: raise TimeoutError return None except Exception as e: # pylint: disable=broad-except logger.error(log_messages.HTTP_REQUEST_RETURNED_ERROR, method, url, e) return None return json.loads(response.data)
def _query_ip_api(self, hostname, dict_result): target_url = "http://ip-api.com/json/" + hostname retries = Retry(connect=MAX_RETRY, read=MAX_RETRY, status=MAX_RETRY, redirect=2, backoff_factor=SEC_FOR_BACKOFF) timeout = Timeout(connect=3.0, read=2.0) worker = PoolManager(retries=retries, timeout=timeout) try: # prevention of DOS if self.last: delta = datetime.now() - self.last delay_offset = delta.total_seconds() if delay_offset < SEC_BETWEEN_DYNAMIC_QUERY: sec2sleep = SEC_BETWEEN_DYNAMIC_QUERY - delay_offset Logger.debug("sleep %f seconds because of setting %f" % (sec2sleep, SEC_BETWEEN_DYNAMIC_QUERY)) sleep(sec2sleep) # action request = worker.request('GET', target_url) if request.status == 200: page = json.loads(request.data) try: for name in dict_result.keys(): if name in page: dict_result[name] = page[name] self.last = datetime.now() except KeyError: Logger.error("Failed parse JSON from GET %s" % target_url) Logger.error(request.data) except urllibs_exceptions as err: if 'message' in dir(err): Logger.error(err.message) else: Logger.error(err.__name__) except JSONDecoderError as err: if 'message' in dir(err): Logger.error(err.message) else: Logger.error(err.__name__)
def request_Api(api_name, params): """ 通用HTTP 发送请求 :param api_name: 接口名 :param params: 参数 :return: """ ("api名字: {} -- 参数: {}".format(api_name, params)) url = "http://39.98.39.224:35645" payload = { 'jsonrpc': '2.0', 'method': api_name, # 'chain_getBalance' 'params': params, # ['0xad3dc2d8aedef155eaba42ab72c1fe480699336c'], 'id': 3 } logging.info("参数:{},payload:{}".format(params, payload)) payload = json.dumps(payload) headers = { 'Content-Type': "application/json", 'cache-control': "no-cache", 'Postman-Token': "27a29181-18f4-4549-80c2-d23196a7df15", 'Connection': "close" } try: requests.adapters.DEFAULT_RETRIES = 5 session = requests.Session() session.keep_alive = False retry = Retry(connect=5, backoff_factor=0.5) adapter = HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) response = session.request("POST", url, data=payload, headers=headers) logging.info(response.text) jsonDic = json.loads(response.text) return jsonDic except Exception as e: logging.error("接口报错{}".format(e)) return e # -1 默认接口调用失败
def __init__(self, api_key=None, proxy_dict=None, env=None, json_encoder=None, adapter=None): if api_key: self._FCM_API_KEY = api_key elif os.getenv('FCM_API_KEY', None): self._FCM_API_KEY = os.getenv('FCM_API_KEY', None) else: raise AuthenticationError( "Please provide the api_key in the google-services.json file") self.FCM_REQ_PROXIES = None self.requests_session = requests.Session() retries = Retry(backoff_factor=1, status_forcelist=[502, 503], allowed_methods=(Retry.DEFAULT_ALLOWED_METHODS | frozenset(['POST']))) self.requests_session.mount( 'http://', adapter or HTTPAdapter(max_retries=retries)) self.requests_session.mount( 'https://', adapter or HTTPAdapter(max_retries=retries)) self.requests_session.headers.update(self.request_headers()) self.requests_session.mount(self.INFO_END_POINT, HTTPAdapter(max_retries=self.INFO_RETRIES)) if proxy_dict and isinstance(proxy_dict, dict) and ( ('http' in proxy_dict) or ('https' in proxy_dict)): self.FCM_REQ_PROXIES = proxy_dict self.requests_session.proxies.update(proxy_dict) self.send_request_responses = [] if env == 'app_engine': try: from requests_toolbelt.adapters import appengine appengine.monkeypatch() except ModuleNotFoundError: pass self.json_encoder = json_encoder
def client( retries=3, backoff_factor=0.3, status_forcelist=(500, 502, 504), session=None, ): session = session or requests.Session() session.verify = bool(int(app.config.get('REQUESTS_VERIFY'))) retry = Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, ) adapter = HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) return session
def set_forceful_retries(self): config.config.retries = Retry( total=10, backoff_factor=0.25, status_forcelist=[400, 500, 501, 502, 503], allowed_methods=[ "HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE", "POST", "PATCH", ], raise_on_status=False, # Don't remove any headers on redirect remove_headers_on_redirect=[], ) self.update_clients()
def retry_session(self): if not hasattr(self, "_retry_session"): self._retry_session = requests.Session() retry = Retry( total=3, read=3, connect=3, backoff_factor=2, status_forcelist=[429, 500, 502, 503, 504], ) adapter = HTTPAdapter(max_retries=retry) self._retry_session.mount("http://", adapter) self._retry_session.mount("https://", adapter) self._threaded_done = 0 self._threaded_exceptions = 0 self._periodic_http_done = 0 self._periodic_http_exceptions = 0 self._periodic_ws_done = 0 self._periodic_ws_exceptions = 0 return self._retry_session
def __init__(self): """A Requests session. + retries + auto raise for status """ super(Session, self).__init__() logging.debug("Setting up retries for a new session...") retry_strategy = Retry( total=requests_extra.defaults.retries_total, backoff_factor=requests_extra.defaults.retries_backoff_factor, status_forcelist=requests_extra.defaults.retries_status_forcelist, allowed_methods=requests_extra.defaults.retries_allowed_methods, ) self.mount("http://", HTTPAdapter(max_retries=retry_strategy)) self.mount("https://", HTTPAdapter(max_retries=retry_strategy)) self.headers = default_headers_with_brotli()
def __init__(self, username, dev=None): self.username = username self.mirror = None # alternate ligand download locations self.minEnergy = -8 # for binding energies lower than this threshold, upload autodock logfile if dev is not None: self.server = SERVER else: self.server = PRODUCTION_SERVER self.apiPath = self.server + '/api/'+API_V # implementation found from : https://stackoverflow.com/questions/23267409/how-to-implement-retry-mechanism-into-python-requests-library self.session = requests.Session() # nginx will conveniently return a 502 if the flask server goes down methods = frozenset(["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE", "POST"] ) retries = Retry(total=5, backoff_factor=1, status_forcelist=[500, 502, 503, 504], method_whitelist=methods) # lets try a 500 code too .... self.session.mount('https://', HTTPAdapter(max_retries=retries))
def __init__(self, async_endpoint=None, api_key=None, user_id=None): self._session = requests.Session() retry_adapter = HTTPAdapter( max_retries=Retry(total=3, method_whitelist=['POST'], status_forcelist=[502, 503, 504, 521, 523], backoff_factor=0.2, raise_on_status=False)) # We need this to get binary payload for the wavefunction call. self._session.headers.update({"Accept": "application/octet-stream"}) self._session.mount("http://", retry_adapter) self._session.mount("https://", retry_adapter) config = PyquilConfig() self.api_key = api_key if api_key else config.api_key self.user_id = user_id if user_id else config.user_id self.async_endpoint = async_endpoint
def __init__(self, client, verify=True): self.client = client self.timeout = self.client.timeout self.session = session() retries = Retry(total=self.client.max_retries, backoff_factor=self.client.backoff_factor) self.session.mount('http://', HTTPAdapter(max_retries=retries)) self.session.mount('https://', HTTPAdapter(max_retries=retries)) self.session.verify = verify #if 'Authorization' not in self.session.headers: # self.authorize() self.session.headers[ 'Authorization'] = 'Token ' + self.client.access_token if self.client.vendor_key: self.session.headers['Vendor-Key'] = self.client.vendor_key
def retry_session(): # This will give the total wait time in minutes: # >>> sum([min((0.3 * (2 ** (i - 1))), 120) / 60 for i in range(24)]) # >>> 30.5575 # This works by the using the minimum time in seconds of the backoff time # and the max back off time which defaults to 120 seconds. The backoff time # increases after every failed attempt. session = requests.Session() retry = Retry( total=3, # 24 read=5, connect=3, # 24 backoff_factor=0.3, status_forcelist=(500, 502, 504), method_whitelist=('GET', 'POST'), ) adapter = HTTPAdapter(max_retries=retry) # session.mount('http://', adapter) session.mount('https://', adapter) return session
def getteamcsq(hostname,username,password,teamid): # this snippet implements backoff timer and retries when hammering the UCCX with api requests session = requests.Session() retry = Retry(connect=3, backoff_factor=0.5) adapter = HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) resources = session.get('https://' + hostname + '/adminapi/team/' + teamid, auth=HTTPBasicAuth(username, password), verify=False) # 200 if success # Creating Tree... root = ET.fromstring(resources.content) # Putting relevant values as a dictionary in a list for improved ease of handling ... csqs = [] # in contrary to findall, iter search in all sub-elements, not only in the directly descending ones of the root for csq in root.iter('csq'): csqs.append(csq.attrib['name']) if not csqs : csqs.append('None') return csqs
def build(current_position_lat, current_position_long, output_path): url = _build_url(current_position_lat, current_position_long) print(url) session = requests.Session() retry = Retry( total=DOWNLOAD_RETRIES, read=DOWNLOAD_RETRIES, connect=DOWNLOAD_RETRIES, backoff_factor=DOWNLOAD_BACKOFF_FACTOR ) adapter = HTTPAdapter(max_retries=retry) session.mount('https://', adapter) response = session.get(url) with open(output_path, 'wb') as output_file: output_file.write(response.content)
def __init__(self, api_base_url=__API_BASE_URL): self.fields = [] self.api_base_url = api_base_url self.request_timeout = 120 self.session = requests.Session() retries = Retry(total=5, backoff_factor=0.5, status_forcelist=[502, 503, 504]) self.session.mount('http://', HTTPAdapter(max_retries=retries)) self.unit = '㎍/㎥' self.units = ":heart_eyes:(좋아요), :thinking_face:(보통) :sob:(나쁨),:scream:(매우나쁨)" self.area_list = [ '서울', '부산', '대구', '인천', '광주', '대전', '울산', '경기', '강원', '충북', '충남', '전북', '전남', '경북', '경남', '제주', '세종' ] self.eng_list = [ "seoul", "busan", "daegu", "incheon", "gwangju", "daejeon", "ulsan", "gyeonggi", "gangwon", "chungbuk", "chungnam", "jeonbuk", "jeonnam", "gyeongbuk", "gyeongnam", "jeju", "sejong" ]
class HTTP(object): retries = 5 redirects = 2 timeout = 120 read = None domain = 'https://api.every-sense.com' port = 8001 headers = {'Content-Type': 'application/json;charset=utf-8'} _conn = PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=where(), timeout=Timeout(connect=timeout, read=read), retries=Retry(connect=retries, redirect=redirects)) def request(self, api_method, body): url = self.domain + ':' + str(self.port) + '/' + api_method try: req = self._conn.urlopen( 'POST', url, headers=self.headers, body=dumps(body), ) res = loads(req.data.decode('utf-8')) return res except exceptions.MaxRetryError as e: res = { 'code': -40, 'reason': 'Max retry (%s) exceeded ' % self.retries } return res except exceptions.ConnectTimeoutError as e: res = { 'code': -30, 'reason': 'Connection timeout (%s sec)' % self.timeout } return res except Exception as e: raise e finally: req.release_conn()
def api_client(): session = Session() retry = Retry(total=0, read=0, connect=0, backoff_factor=1, status_forcelist=STATUS_FORCELIST) adapter = HTTPAdapter(max_retries=retry) session.mount("http://", adapter) session.mount("https://", adapter) client = APIClient( request_session=session, project="test_proj", base_url="http://localtest.com/api/0.5/projects/test_proj", num_of_workers=1, cookies={"a-cookie": "a-cookie-val"}, headers={}, timeout=60, ) yield client
def requests_retry_session(retries=6, backoff_factor=1, status_forcelist=(500, 502, 504), session=None): """ Creates a request session that has auto retry. Implementation by Adam @mcadm34 https://gitlab.com/thorchain/devops/node-launcher/-/blob/master/telegram-bot/templates/configmap.yaml#L192-212 """ session = session or requests.Session() retry = Retry(total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist) adapter = HTTPAdapter(max_retries=retry) session.mount("http://", adapter) session.mount("https://", adapter) return session