def get_requests_session(self):
requests_session = self._session.GetParameter("requests_session")
if requests_session == None:
# To make sure we can use the requests session in the threadpool we
# need to make sure that the connection pool can block. Otherwise it
# will raise when it runs out of connections and the threads will be
# terminated.
requests_session = requests.Session()
requests_session.mount(
"https://",
adapters.HTTPAdapter(pool_connections=10,
pool_maxsize=300,
max_retries=10,
pool_block=True))
requests_session.mount(
"http://",
adapters.HTTPAdapter(pool_connections=10,
pool_maxsize=300,
max_retries=10,
pool_block=True))
self._session.SetCache("requests_session", requests_session)
return requests_session
def _create_http_session(self):
self.session = Session()
if self.headers:
self.session.headers.update(self.headers)
self.session.mount('http://',
adapters.HTTPAdapter(max_retries=self.max_retries))
self.session.mount('https://',
adapters.HTTPAdapter(max_retries=self.max_retries))
def __init__(self):
self.session = requests.session()
self.session.keep_alive = False
self.session.mount(
'https://',
adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100))
self.session.mount(
'http://',
adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100))
self.ua = UserAgent()
def __init__(self, CONNECT_TIMEOUT=CONNECT_TIMEOUT, READ_TIMEOUT=READ_TIMEOUT, stream=False):
self.requests = Session()
self.requests.stream = stream
self.requests.trust_env = False
self.requests.mount('http://', adapters.HTTPAdapter(pool_connections=NUM_POOLS,
pool_maxsize=POOL_MAXSIZE))
self.requests.mount('https://', adapters.HTTPAdapter(pool_connections=NUM_POOLS,
pool_maxsize=POOL_MAXSIZE))
self.tuple = (CONNECT_TIMEOUT, READ_TIMEOUT)
def new_session(insecure=False, ca_file=None, total_retries=None):
session = requests.Session()
if total_retries is not None:
http_adapter = adapters.HTTPAdapter(max_retries=retry.Retry(
total=total_retries))
https_adapter = adapters.HTTPAdapter(max_retries=retry.Retry(
total=total_retries))
session.mount('http://', http_adapter)
session.mount('https://', https_adapter)
session.verify = ca_file if ca_file else not insecure
return session
def get_discovery_key_session():
if RequestsHelper._discovery_key_session is None:
print('Creating new requests session')
session = Session()
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504])
session.mount('https://',
adapters.HTTPAdapter(max_retries=retries))
session.mount('http://', adapters.HTTPAdapter(max_retries=retries))
RequestsHelper._discovery_key_session = session
return RequestsHelper._discovery_key_session
def __init__(self):
requests.adapters.DEFAULT_RETRIES = 5
self.session = requests.session()
self.session.keep_alive = False
self.session.mount(
'https://',
adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100))
self.session.mount(
'http://',
adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100))
self.ua = UserAgent()
self.pool = IP_Pool()
self.pool.pull_ip()
def __init__(self, max_retries=3):
self.session = Session()
adapter = adapters.HTTPAdapter(max_retries=max_retries)
self.session.mount('http://', adapter)
self.session.mount('https://', adapter)
self.session.keep_alive = False
self.cookies = {}
def _init_session(self):
sess = HTMLSession()
adapter = adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)
sess.mount('http://', adapter)
sess.mount('https://', adapter)
return sess
def test_precached_session(mocker):
mocker.patch("requests.Session.post", return_value=MockedResponse())
# Update the cache with a handcrafted session
adapter = adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)
session = Session()
session.mount('http://', adapter)
session.mount('https://', adapter)
request.cache_session(URI, session)
# Submit a second request with different arguments
assert len(request._session_cache) == 1
response = request.make_post_request(URI, b'request', timeout=60)
assert response == "content"
assert len(request._session_cache) == 1
# Ensure the timeout was passed to the request
session = request._get_session(URI)
session.post.assert_called_once_with(URI, data=b'request', timeout=60)
# Ensure the adapter parameters match those we specified
check_adapters_mounted(session)
adapter = session.get_adapter(URI)
assert isinstance(adapter, HTTPAdapter)
assert adapter._pool_connections == 100
assert adapter._pool_maxsize == 100
def __init__(self, base_url: str, *args, **kwargs):
"""
Creates a new HTTP session instance.
Args:
base_url: The base URL for this HTTP session
*args: The optional positional arguments
**kwargs: The optional keyword arguments
"""
super().__init__(*args, **kwargs)
self._base_url = base_url
# Custom headers
self.headers.update({
'User-Agent':
'geodatabr/{version} ({url})'.format(version=__meta__.__version__,
url=__meta__.__url__),
})
# Automatic retries
for protocol in ('http://', 'https://'):
self.mount(
protocol,
adapters.HTTPAdapter(max_retries=retry.Retry(
total=HTTP_MAX_RETRIES,
backoff_factor=HTTP_BACKOFF_FACTOR,
status_forcelist=HTTP_RETRY_STATUSES)))
def __init__(self,
auth_callback=None,
session=None,
client_id=None,
client_secret=None,
access_token=None,
access_token_cache_file=None):
self._res = {}
self.auth_callback = auth_callback
self.pin = None
self._access_token_cache_file = access_token_cache_file
self._client_id = client_id
self._client_secret = client_secret
self._access_token = access_token
if (access_token_cache_file is not None and access_token is None
and os.path.exists(access_token_cache_file)):
with open(access_token_cache_file, 'r') as f:
self._res = json.load(f)
self._callback(self._res)
if session is not None:
session = weakref.ref(session)
self._session = session
self._adapter = adapters.HTTPAdapter()
def __init__(self, config, logger):
self._logutils = LogUtils(logger)
self._config = config
self._url = config.get_service_url()
self._request_uri = self._url.geturl() + HttpConstants.NOSQL_DATA_PATH
self._pool_connections = config.get_pool_connections()
self._pool_maxsize = config.get_pool_maxsize()
self._max_request_id = 1
self._proxy_host = config.get_proxy_host()
self._proxy_port = config.get_proxy_port()
self._proxy_username = config.get_proxy_username()
self._proxy_password = config.get_proxy_password()
self._retry_handler = config.get_retry_handler()
if self._retry_handler is None:
self._retry_handler = DefaultRetryHandler()
self._sec_info_timeout = config.get_sec_info_timeout()
self._shut_down = False
self._user_agent = self._make_user_agent()
self._auth_provider = config.get_authorization_provider()
if self._auth_provider is None:
raise IllegalArgumentException(
'Must configure AuthorizationProvider.')
self._sess = Session()
adapter = adapters.HTTPAdapter(pool_connections=self._pool_connections,
pool_maxsize=self._pool_maxsize,
max_retries=5, pool_block=True)
self._sess.mount(self._url.scheme + '://', adapter)
if self._proxy_host is not None:
self._check_and_set_proxy(self._sess)
def new_connection(self, cluster_api, provider):
config = cluster_api.nsxlib_config
session = TimeoutSession(config.http_timeout, config.http_read_timeout)
if config.client_cert_provider:
session.cert_provider = config.client_cert_provider
else:
session.auth = (provider.username, provider.password)
# NSX v3 doesn't use redirects
session.max_redirects = 0
session.verify = not config.insecure
if session.verify and provider.ca_file:
# verify using the said ca bundle path
session.verify = provider.ca_file
# we are pooling with eventlet in the cluster class
adapter = adapters.HTTPAdapter(pool_connections=1,
pool_maxsize=1,
max_retries=config.retries,
pool_block=False)
session.mount('http://', adapter)
session.mount('https://', adapter)
self.get_default_headers(session, provider,
config.allow_overwrite_header)
return session
def _make_adapter_with_retries() -> adapters.HTTPAdapter:
retry = urllib3.Retry(
connect=5,
total=10,
method_whitelist=["GET"],
status_forcelist=[500, 501, 502, 503],
)
return adapters.HTTPAdapter(max_retries=retry)
def __init__(self, *args, **kwargs):
super(Consumer, self).__init__(*args, **kwargs)
if not self.type:
self.type = self.TYPE
# attach session
self.session = requests.Session()
adapter = adapters.HTTPAdapter(max_retries=3)
self.session.mount('http://', adapter)
def query_api(full_url):
session = requests.Session()
for scheme in ('http://', 'https://'):
session.mount(
scheme, adapters.HTTPAdapter(max_retries=retry.Retry(connect=5), ))
r_s = session.get(full_url)
return r_s
def make_session(http_max_pool_size, num_retries):
session = grequests.Session()
adapter = adapters.HTTPAdapter(max_retries=num_retries,
pool_maxsize=http_max_pool_size,
pool_block=True)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def _adapter_pool(url):
"""
Store and reuse HTTP adapters per Service URL.
"""
if url not in _ADAPTERS:
_ADAPTERS[url] = adapters.HTTPAdapter()
return _ADAPTERS[url]
def _make_adapter_with_retries():
retry = util.Retry(
connect=5,
total=10,
method_whitelist=['GET'],
status_forcelist=[500, 501, 502, 503],
)
return adapters.HTTPAdapter(max_retries=retry)
def get(self, url):
"""
Store and reuse HTTP adapters per Service URL.
"""
if url not in self._adapters:
self._adapters[url] = adapters.HTTPAdapter()
return self._adapters[url]
def create(self):
"""
Create the requests.Session to communicate with the RESTful API.
:return: :const:`None`
:rtype: NoneType
"""
if self._session:
self.close()
if not self._session:
self._session = requests.Session()
self._session.mount('http://',
ra.HTTPAdapter(max_retries=self._max_retries))
self._session.mount('https://',
ra.HTTPAdapter(max_retries=self._max_retries))
msg = u'Created internal requests Session instance {0:#0x}'
log_with_debug_info(logging.DEBUG, msg.format(id(self._session)))
def start_http_session() -> requests.Session:
"""Открытие клиентского соединение с интернетом."""
# logging.info("Открывается сессия для обновления данных по интернет")
session = requests.Session()
adapter = adapters.HTTPAdapter(pool_maxsize=HTTPS_MAX_POOL_SIZE,
max_retries=MAX_RETRIES,
pool_block=True)
session.mount("https://", adapter)
return session
def adapter(self):
retry = util.Retry(
connect=5,
total=10,
method_whitelist=["GET"],
status_forcelist=[500, 501, 502, 503],
)
return adapters.HTTPAdapter(max_retries=retry)
def upload_ptpb(rootpath, filename, notify_bool=True, name="File"):
"""Upload file to ptpb.pw and notify via notify-send.
Keyword arguments:
rootpath -- root path for file.
filename -- name of file.
notify_bool -- boolean to send notification (default True)
name -- name used to generate nicer notifications (default File).
"""
from requests import Session, adapters, exceptions
filepath = os.path.join(rootpath, filename)
if not os.path.isfile(filepath):
print("File not found in '" + filepath + "'.")
return (False)
url = "https://ptpb.pw"
files = {'c': open(filepath, 'rb')}
opts = {'sunset': '432000'}
s = Session()
a = adapters.HTTPAdapter(max_retries=3)
s.mount('https://', a)
try:
rh = s.get(url)
except exceptions.RequestException as e:
print("Error: Failed to connect to " + url)
notify(
name + ":", name + " '" + filename + "'could not be uploaded." +
" Failed to connect to " + url)
return False
if not (rh.status_code == 200):
print("Error: Failed to connect to " + url)
print(rh.status_code)
if notify_bool:
notify(
name + ":", name + " " + filename + "could not be uploaded." +
" Failed to connect to " + url)
return False
r = s.post(url + "/?u=1", files=files, data=opts)
if (r.status_code == 200):
pasteurl = r.headers['Location']
if notify_bool:
notify(name + ":",
name + " " + filename + " uploaded to:" + " " + pasteurl)
else:
if notify_bool:
notify(
name + ":", name + " " + filename + "could not be uploaded." +
" Status code: " + r.status_code)
print("Error: Upload was unsuccessful.")
print(r.status_code)
return False
return (pasteurl)
def _init_values(self, target):
self._folder = target
# if target and not self.sin_sub:
# self._folder = self._create_folders(target)
self._emitidas = False
self._current_year = datetime.datetime.now().year
self._session = Session()
a = adapters.HTTPAdapter(pool_connections=512, pool_maxsize=512, max_retries=5)
self._session.mount('https://', a)
return
def initclass(cls):
if not cls._initialized:
cls.logger = logging.getLogger(cls.__module__)
cls._session = Session()
adapter = adapters.HTTPAdapter(max_retries=5)
cls._session.mount('http://', adapter)
cls._session.mount('https://', adapter)
cls._reqlock = Lock()
cls._initialized = True
cls.logger.debug('Initializing ' + cls.__name__)
def __init__(self, server_http_url="", *args, **kwargs):
self.server_http_url = server_http_url
self.session = requests.session()
self.session.headers = {"Accept": "application/json", "Content-type": "application/json"}
self.session.verify = False
# Increase the number of pool connections so we can test large numbers
# of connections to the api.
adapter = adapters.HTTPAdapter(pool_connections=2000, pool_maxsize=2000)
self.session.mount("http://", adapter)
def __init__(self,
time_out=30,
pool_num=10,
pool_max_size=50,
max_retries=3):
super(SyncClientSession, self).__init__()
self._time_out = time_out
self._pool_num = pool_num
self._pool_max_size = pool_max_size
self._max_retries = max_retries
self.mount(
"http://",
adapters.HTTPAdapter(pool_connections=self._pool_num,
pool_maxsize=self._pool_max_size,
max_retries=self._max_retries))
self.mount(
"https://",
adapters.HTTPAdapter(pool_connections=self._pool_num,
pool_maxsize=self._pool_max_size,
max_retries=self._max_retries))
def _build_adapters(self):
"""Parse adapter_settings to build adapters and mount within this session
If retry settings are included, a retry object is built for the adapter.
"""
for prefix, settings in self.adapter_settings.items():
adapter = adapters.HTTPAdapter()
if 'adapter' in settings:
if 'retry' in settings:
adapter = adapters.HTTPAdapter(
max_retries=Refry(**settings['retry']),
**settings['adapter']
)
else:
adapter = adapters.HTTPAdapter(**settings['adapter'])
adapter.config.update(settings.get('adapter_config', {}))
self.mount(prefix, adapter)
