def requests_session():
# Setup a session with a retry policy
session = requests.Session()
adapter = HTTPAdapter(
max_retries=Retry(
total=_UPLOAD_MAX_TRIES,
backoff_factor=_UPLOAD_RETRY_BACKOFF,
status_forcelist=[500, 502, 503, 504, 104],
)
)
# Override longest-match adapter for http and https
session.mount(
"http://", adapter,
)
session.mount(
"https://", adapter,
)
yield session
def __init__(self, host, user, password):
retry_strategy = Retry(
total=10,
# status_forcelist=[429, 500, 502, 503, 504],
method_whitelist=["HEAD", "GET", "OPTIONS"],
backoff_factor=1
)
adapter = HTTPAdapter(max_retries=retry_strategy)
self.http = requests.Session()
self.http.mount("https://", adapter)
self.http.mount("http://", adapter)
self._auth = HTTPBasicAuth(user, password)
self.http.auth = self._auth
if host.startswith("http"):
self._conn_string = host
else:
self._conn_string = "http://%s" % host
def __init__(self):
self.oauth_url = "https://api.real-debrid.com/oauth/v2/"
self.device_code_url = "device/code?{}"
self.device_credentials_url = "device/credentials?{}"
self.token_url = "token"
self.device_code = ""
self.oauth_timeout = 0
self.oauth_time_step = 0
self.base_url = "https://api.real-debrid.com/rest/1.0/"
self.cache_check_results = {}
self.progress_dialog = xbmcgui.DialogProgress()
self.session = requests.Session()
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504])
self.session.mount("https://",
HTTPAdapter(max_retries=retries, pool_maxsize=100))
self._load_settings()
def requests_retry_session(
retries=3,
backoff_factor=0.3,
status_forcelist=(500, 502, 504),
session=None,
):
session = session or requests.Session()
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def get_http_session(retries=3,
backoff_factor=0.3,
status_forcelist=(500, 502, 504)) -> requests.Session:
"""
Taken from: https://www.peterbe.com/plog/best-practice-with-retries-with-requests
"""
session = requests.Session()
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def setup_configuration(api_key, host, username="", password="", verify_ssl=True):
host = host + "/v1"
global config
config.api_key = api_key
config.host = host
config.username = username
config.password = password
config.verify_ssl = verify_ssl
config.retries = Retry(
total=10,
backoff_factor=0.25,
status_forcelist=[503],
method_whitelist=False,
raise_on_status=False,
)
# Set logged in at this point
global logged_in
logged_in = True
def __init__(
self,
jira_url,
auth,
params=None,
webbrowser_auth=False,
jsessionid_for_testing=None,
):
retries = 3
backoff_factor = 0.3
status_forcelist = (500, 502, 504)
retry_session = None
self.jira_url = jira_url
self.insight_api_url = f"{jira_url}/rest/insight/1.0"
self.auth = auth
# Configure retry session
self.retry_session = retry_session or requests.Session()
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
self.retry_session.mount("http://", adapter)
self.retry_session.mount("https://", adapter)
self.retry_session.auth = self.auth
self.retry_session.params = params
# TODO: Find out, why the lib does not extract session cookies
if webbrowser_auth:
from urllib.parse import urlparse
url = urlparse(self.jira_url)
import browser_cookie3
cookies = browser_cookie3.firefox(domain_name=url.netloc)
self.retry_session.cookies = cookies
if jsessionid_for_testing:
self.retry_session.cookies = requests.cookies.cookiejar_from_dict(
{"JSESSIONID": jsessionid_for_testing}
)
def get_retry_policy(num_retry=3):
"""
:return: Returns the msrest or requests REST client retry policy.
:rtype: urllib3.Retry
"""
status_forcelist = [413, 429, 500, 502, 503, 504]
backoff_factor = 0.4
retry_policy = Retry(
total=num_retry,
read=num_retry,
connect=num_retry,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
# By default this is True. We set it to false to get the full error trace, including url and
# status code of the last retry. Otherwise, the error message is 'too many 500 error responses',
# which is not useful.
raise_on_status=False,
)
return retry_policy
def requests_retry_session(
# Thank you Peter Becom
# https://www.peterbe.com/plog/best-practice-with-retries-with-requests
retries=3,
backoff_factor=5,
status_forcelist=(500, 502, 504),
session=None):
session = session or requests.Session()
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def doi2bib(doi):
retry_strategy = Retry(total=3,
status_forcelist=[429, 500, 502, 503, 504],
method_whitelist=["HEAD", "GET", "OPTIONS"])
adapter = HTTPAdapter(max_retries=retry_strategy)
http = requests.Session()
http.mount("https://", adapter)
http.mount("http://", adapter)
url = f"http://dx.doi.org/{doi}"
headers = {'Accept': 'application/x-bibtex'}
logging.info(f"Retrieving bibtex from {url}")
r = http.get(url, headers=headers)
# r = requests.get(url, headers=headers)
if r.status_code is not 200:
logging.info(r)
raise RequestException("Failed to retrieve BIB for a DOI.")
return r.text
def ensure_session(self, establish_retries=0):
try:
if establish_retries > 5:
# maximum retries to login (get uid)
# connecting retires specified below
return False
if self.get_uid(self.username):
return True
else:
self.session = requests.Session()
self.session.mount(
'https://',
HTTPAdapter(
max_retries=Retry(total=5, backoff_factor=0.1)))
return self.ensure_session(establish_retries + 1)
except:
traceback.print_exc()
return False
def _requests_retry_session(
# A function for basic retrying of an url when it isn't accessible immediately.
retries=8,
backoff_factor=0.01,
status_forcelist=(500, 502, 504),
session=None,
):
session = session or requests.Session()
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def _http_request(method: str, endpoint: str, handle_timeout: bool,
**kwargs) -> Optional[JT]:
url = build_url(endpoint)
parsed_url = parse_url(url)
pm_args = {
"num_pools":
constants.HTTP_POOL_MANAGER_COUNT,
"host":
parsed_url.host,
"port":
parsed_url.port,
"retries":
Retry(connect=constants.HTTP_REQUEST_RETRIES_COUNT,
read=constants.HTTP_REQUEST_RETRIES_COUNT,
redirect=constants.HTTP_REQUEST_RETRIES_COUNT,
backoff_factor=constants.HTTP_REQUEST_BACKOFF_FACTOR,
method_whitelist=METHODS_WHITELIST),
"ssl_context":
_ssl_context,
}
if _ssl_context is not None and url.startswith("https"):
pm_args["assert_hostname"] = False
http_pool_manager: PoolManager = PoolManager(**pm_args)
try:
logger.trace("HTTP {0} to {1}", method, url)
response = http_pool_manager.request(
method=method,
url=parsed_url.url,
timeout=constants.HTTP_REQUEST_TIMEOUT,
**kwargs)
raise_for_status(response)
except MaxRetryError as e:
logger.info("{} to {} failed due to: {}.", method, url, e)
return None
except TimeoutError:
if handle_timeout:
raise TimeoutError
return None
except Exception as e: # pylint: disable=broad-except
logger.error(log_messages.HTTP_REQUEST_RETURNED_ERROR, method, url, e)
return None
return json.loads(response.data)
def _query_ip_api(self, hostname, dict_result):
target_url = "http://ip-api.com/json/" + hostname
retries = Retry(connect=MAX_RETRY,
read=MAX_RETRY,
status=MAX_RETRY,
redirect=2,
backoff_factor=SEC_FOR_BACKOFF)
timeout = Timeout(connect=3.0, read=2.0)
worker = PoolManager(retries=retries, timeout=timeout)
try:
# prevention of DOS
if self.last:
delta = datetime.now() - self.last
delay_offset = delta.total_seconds()
if delay_offset < SEC_BETWEEN_DYNAMIC_QUERY:
sec2sleep = SEC_BETWEEN_DYNAMIC_QUERY - delay_offset
Logger.debug("sleep %f seconds because of setting %f" %
(sec2sleep, SEC_BETWEEN_DYNAMIC_QUERY))
sleep(sec2sleep)
# action
request = worker.request('GET', target_url)
if request.status == 200:
page = json.loads(request.data)
try:
for name in dict_result.keys():
if name in page:
dict_result[name] = page[name]
self.last = datetime.now()
except KeyError:
Logger.error("Failed parse JSON from GET %s" % target_url)
Logger.error(request.data)
except urllibs_exceptions as err:
if 'message' in dir(err):
Logger.error(err.message)
else:
Logger.error(err.__name__)
except JSONDecoderError as err:
if 'message' in dir(err):
Logger.error(err.message)
else:
Logger.error(err.__name__)
def request_Api(api_name, params):
"""
通用HTTP 发送请求
:param api_name: 接口名
:param params: 参数
:return:
"""
("api名字: {} -- 参数: {}".format(api_name, params))
url = "http://39.98.39.224:35645"
payload = {
'jsonrpc': '2.0',
'method': api_name, # 'chain_getBalance'
'params': params, # ['0xad3dc2d8aedef155eaba42ab72c1fe480699336c'],
'id': 3
}
logging.info("参数:{},payload:{}".format(params, payload))
payload = json.dumps(payload)
headers = {
'Content-Type': "application/json",
'cache-control': "no-cache",
'Postman-Token': "27a29181-18f4-4549-80c2-d23196a7df15",
'Connection': "close"
}
try:
requests.adapters.DEFAULT_RETRIES = 5
session = requests.Session()
session.keep_alive = False
retry = Retry(connect=5, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
response = session.request("POST", url, data=payload, headers=headers)
logging.info(response.text)
jsonDic = json.loads(response.text)
return jsonDic
except Exception as e:
logging.error("接口报错{}".format(e))
return e # -1 默认接口调用失败
def __init__(self,
api_key=None,
proxy_dict=None,
env=None,
json_encoder=None,
adapter=None):
if api_key:
self._FCM_API_KEY = api_key
elif os.getenv('FCM_API_KEY', None):
self._FCM_API_KEY = os.getenv('FCM_API_KEY', None)
else:
raise AuthenticationError(
"Please provide the api_key in the google-services.json file")
self.FCM_REQ_PROXIES = None
self.requests_session = requests.Session()
retries = Retry(backoff_factor=1,
status_forcelist=[502, 503],
allowed_methods=(Retry.DEFAULT_ALLOWED_METHODS
| frozenset(['POST'])))
self.requests_session.mount(
'http://', adapter or HTTPAdapter(max_retries=retries))
self.requests_session.mount(
'https://', adapter or HTTPAdapter(max_retries=retries))
self.requests_session.headers.update(self.request_headers())
self.requests_session.mount(self.INFO_END_POINT,
HTTPAdapter(max_retries=self.INFO_RETRIES))
if proxy_dict and isinstance(proxy_dict, dict) and (
('http' in proxy_dict) or ('https' in proxy_dict)):
self.FCM_REQ_PROXIES = proxy_dict
self.requests_session.proxies.update(proxy_dict)
self.send_request_responses = []
if env == 'app_engine':
try:
from requests_toolbelt.adapters import appengine
appengine.monkeypatch()
except ModuleNotFoundError:
pass
self.json_encoder = json_encoder
def client(
retries=3,
backoff_factor=0.3,
status_forcelist=(500, 502, 504),
session=None,
):
session = session or requests.Session()
session.verify = bool(int(app.config.get('REQUESTS_VERIFY')))
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def set_forceful_retries(self):
config.config.retries = Retry(
total=10,
backoff_factor=0.25,
status_forcelist=[400, 500, 501, 502, 503],
allowed_methods=[
"HEAD",
"GET",
"PUT",
"DELETE",
"OPTIONS",
"TRACE",
"POST",
"PATCH",
],
raise_on_status=False,
# Don't remove any headers on redirect
remove_headers_on_redirect=[],
)
self.update_clients()
def retry_session(self):
if not hasattr(self, "_retry_session"):
self._retry_session = requests.Session()
retry = Retry(
total=3,
read=3,
connect=3,
backoff_factor=2,
status_forcelist=[429, 500, 502, 503, 504],
)
adapter = HTTPAdapter(max_retries=retry)
self._retry_session.mount("http://", adapter)
self._retry_session.mount("https://", adapter)
self._threaded_done = 0
self._threaded_exceptions = 0
self._periodic_http_done = 0
self._periodic_http_exceptions = 0
self._periodic_ws_done = 0
self._periodic_ws_exceptions = 0
return self._retry_session
def __init__(self):
"""A Requests session.
+ retries
+ auto raise for status
"""
super(Session, self).__init__()
logging.debug("Setting up retries for a new session...")
retry_strategy = Retry(
total=requests_extra.defaults.retries_total,
backoff_factor=requests_extra.defaults.retries_backoff_factor,
status_forcelist=requests_extra.defaults.retries_status_forcelist,
allowed_methods=requests_extra.defaults.retries_allowed_methods,
)
self.mount("http://", HTTPAdapter(max_retries=retry_strategy))
self.mount("https://", HTTPAdapter(max_retries=retry_strategy))
self.headers = default_headers_with_brotli()
def __init__(self, username, dev=None):
self.username = username
self.mirror = None # alternate ligand download locations
self.minEnergy = -8 # for binding energies lower than this threshold, upload autodock logfile
if dev is not None:
self.server = SERVER
else:
self.server = PRODUCTION_SERVER
self.apiPath = self.server + '/api/'+API_V
# implementation found from : https://stackoverflow.com/questions/23267409/how-to-implement-retry-mechanism-into-python-requests-library
self.session = requests.Session()
# nginx will conveniently return a 502 if the flask server goes down
methods = frozenset(["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE", "POST"] )
retries = Retry(total=5, backoff_factor=1, status_forcelist=[500, 502, 503, 504], method_whitelist=methods) # lets try a 500 code too ....
self.session.mount('https://', HTTPAdapter(max_retries=retries))
def __init__(self, async_endpoint=None, api_key=None, user_id=None):
self._session = requests.Session()
retry_adapter = HTTPAdapter(
max_retries=Retry(total=3,
method_whitelist=['POST'],
status_forcelist=[502, 503, 504, 521, 523],
backoff_factor=0.2,
raise_on_status=False))
# We need this to get binary payload for the wavefunction call.
self._session.headers.update({"Accept": "application/octet-stream"})
self._session.mount("http://", retry_adapter)
self._session.mount("https://", retry_adapter)
config = PyquilConfig()
self.api_key = api_key if api_key else config.api_key
self.user_id = user_id if user_id else config.user_id
self.async_endpoint = async_endpoint
def __init__(self, client, verify=True):
self.client = client
self.timeout = self.client.timeout
self.session = session()
retries = Retry(total=self.client.max_retries,
backoff_factor=self.client.backoff_factor)
self.session.mount('http://', HTTPAdapter(max_retries=retries))
self.session.mount('https://', HTTPAdapter(max_retries=retries))
self.session.verify = verify
#if 'Authorization' not in self.session.headers:
# self.authorize()
self.session.headers[
'Authorization'] = 'Token ' + self.client.access_token
if self.client.vendor_key:
self.session.headers['Vendor-Key'] = self.client.vendor_key
def retry_session():
# This will give the total wait time in minutes:
# >>> sum([min((0.3 * (2 ** (i - 1))), 120) / 60 for i in range(24)])
# >>> 30.5575
# This works by the using the minimum time in seconds of the backoff time
# and the max back off time which defaults to 120 seconds. The backoff time
# increases after every failed attempt.
session = requests.Session()
retry = Retry(
total=3, # 24
read=5,
connect=3, # 24
backoff_factor=0.3,
status_forcelist=(500, 502, 504),
method_whitelist=('GET', 'POST'),
)
adapter = HTTPAdapter(max_retries=retry)
# session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def getteamcsq(hostname,username,password,teamid):
# this snippet implements backoff timer and retries when hammering the UCCX with api requests
session = requests.Session()
retry = Retry(connect=3, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
resources = session.get('https://' + hostname + '/adminapi/team/' + teamid, auth=HTTPBasicAuth(username, password), verify=False)
# 200 if success
# Creating Tree...
root = ET.fromstring(resources.content)
# Putting relevant values as a dictionary in a list for improved ease of handling ...
csqs = []
# in contrary to findall, iter search in all sub-elements, not only in the directly descending ones of the root
for csq in root.iter('csq'):
csqs.append(csq.attrib['name'])
if not csqs :
csqs.append('None')
return csqs
def build(current_position_lat, current_position_long, output_path):
url = _build_url(current_position_lat, current_position_long)
print(url)
session = requests.Session()
retry = Retry(
total=DOWNLOAD_RETRIES,
read=DOWNLOAD_RETRIES,
connect=DOWNLOAD_RETRIES,
backoff_factor=DOWNLOAD_BACKOFF_FACTOR
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('https://', adapter)
response = session.get(url)
with open(output_path, 'wb') as output_file:
output_file.write(response.content)
def __init__(self, api_base_url=__API_BASE_URL):
self.fields = []
self.api_base_url = api_base_url
self.request_timeout = 120
self.session = requests.Session()
retries = Retry(total=5,
backoff_factor=0.5,
status_forcelist=[502, 503, 504])
self.session.mount('http://', HTTPAdapter(max_retries=retries))
self.unit = '㎍/㎥'
self.units = ":heart_eyes:(좋아요), :thinking_face:(보통) :sob:(나쁨),:scream:(매우나쁨)"
self.area_list = [
'서울', '부산', '대구', '인천', '광주', '대전', '울산', '경기', '강원', '충북', '충남',
'전북', '전남', '경북', '경남', '제주', '세종'
]
self.eng_list = [
"seoul", "busan", "daegu", "incheon", "gwangju", "daejeon",
"ulsan", "gyeonggi", "gangwon", "chungbuk", "chungnam", "jeonbuk",
"jeonnam", "gyeongbuk", "gyeongnam", "jeju", "sejong"
]
class HTTP(object):
retries = 5
redirects = 2
timeout = 120
read = None
domain = 'https://api.every-sense.com'
port = 8001
headers = {'Content-Type': 'application/json;charset=utf-8'}
_conn = PoolManager(cert_reqs='CERT_REQUIRED',
ca_certs=where(),
timeout=Timeout(connect=timeout, read=read),
retries=Retry(connect=retries, redirect=redirects))
def request(self, api_method, body):
url = self.domain + ':' + str(self.port) + '/' + api_method
try:
req = self._conn.urlopen(
'POST',
url,
headers=self.headers,
body=dumps(body),
)
res = loads(req.data.decode('utf-8'))
return res
except exceptions.MaxRetryError as e:
res = {
'code': -40,
'reason': 'Max retry (%s) exceeded ' % self.retries
}
return res
except exceptions.ConnectTimeoutError as e:
res = {
'code': -30,
'reason': 'Connection timeout (%s sec)' % self.timeout
}
return res
except Exception as e:
raise e
finally:
req.release_conn()
def api_client():
session = Session()
retry = Retry(total=0,
read=0,
connect=0,
backoff_factor=1,
status_forcelist=STATUS_FORCELIST)
adapter = HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
client = APIClient(
request_session=session,
project="test_proj",
base_url="http://localtest.com/api/0.5/projects/test_proj",
num_of_workers=1,
cookies={"a-cookie": "a-cookie-val"},
headers={},
timeout=60,
)
yield client
def requests_retry_session(retries=6,
backoff_factor=1,
status_forcelist=(500, 502, 504),
session=None):
"""
Creates a request session that has auto retry.
Implementation by Adam @mcadm34
https://gitlab.com/thorchain/devops/node-launcher/-/blob/master/telegram-bot/templates/configmap.yaml#L192-212
"""
session = session or requests.Session()
retry = Retry(total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist)
adapter = HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
