def get_requests_session(self): requests_session = self._session.GetParameter("requests_session") if requests_session == None: # To make sure we can use the requests session in the threadpool we # need to make sure that the connection pool can block. Otherwise it # will raise when it runs out of connections and the threads will be # terminated. requests_session = requests.Session() requests_session.mount( "https://", adapters.HTTPAdapter(pool_connections=10, pool_maxsize=300, max_retries=10, pool_block=True)) requests_session.mount( "http://", adapters.HTTPAdapter(pool_connections=10, pool_maxsize=300, max_retries=10, pool_block=True)) self._session.SetCache("requests_session", requests_session) return requests_session
def _create_http_session(self): self.session = Session() if self.headers: self.session.headers.update(self.headers) self.session.mount('http://', adapters.HTTPAdapter(max_retries=self.max_retries)) self.session.mount('https://', adapters.HTTPAdapter(max_retries=self.max_retries))
def __init__(self): self.session = requests.session() self.session.keep_alive = False self.session.mount( 'https://', adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)) self.session.mount( 'http://', adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)) self.ua = UserAgent()
def __init__(self, CONNECT_TIMEOUT=CONNECT_TIMEOUT, READ_TIMEOUT=READ_TIMEOUT, stream=False): self.requests = Session() self.requests.stream = stream self.requests.trust_env = False self.requests.mount('http://', adapters.HTTPAdapter(pool_connections=NUM_POOLS, pool_maxsize=POOL_MAXSIZE)) self.requests.mount('https://', adapters.HTTPAdapter(pool_connections=NUM_POOLS, pool_maxsize=POOL_MAXSIZE)) self.tuple = (CONNECT_TIMEOUT, READ_TIMEOUT)
def new_session(insecure=False, ca_file=None, total_retries=None): session = requests.Session() if total_retries is not None: http_adapter = adapters.HTTPAdapter(max_retries=retry.Retry( total=total_retries)) https_adapter = adapters.HTTPAdapter(max_retries=retry.Retry( total=total_retries)) session.mount('http://', http_adapter) session.mount('https://', https_adapter) session.verify = ca_file if ca_file else not insecure return session
def get_discovery_key_session(): if RequestsHelper._discovery_key_session is None: print('Creating new requests session') session = Session() retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) session.mount('https://', adapters.HTTPAdapter(max_retries=retries)) session.mount('http://', adapters.HTTPAdapter(max_retries=retries)) RequestsHelper._discovery_key_session = session return RequestsHelper._discovery_key_session
def __init__(self): requests.adapters.DEFAULT_RETRIES = 5 self.session = requests.session() self.session.keep_alive = False self.session.mount( 'https://', adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)) self.session.mount( 'http://', adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)) self.ua = UserAgent() self.pool = IP_Pool() self.pool.pull_ip()
def __init__(self, max_retries=3): self.session = Session() adapter = adapters.HTTPAdapter(max_retries=max_retries) self.session.mount('http://', adapter) self.session.mount('https://', adapter) self.session.keep_alive = False self.cookies = {}
def _init_session(self): sess = HTMLSession() adapter = adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100) sess.mount('http://', adapter) sess.mount('https://', adapter) return sess
def test_precached_session(mocker): mocker.patch("requests.Session.post", return_value=MockedResponse()) # Update the cache with a handcrafted session adapter = adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100) session = Session() session.mount('http://', adapter) session.mount('https://', adapter) request.cache_session(URI, session) # Submit a second request with different arguments assert len(request._session_cache) == 1 response = request.make_post_request(URI, b'request', timeout=60) assert response == "content" assert len(request._session_cache) == 1 # Ensure the timeout was passed to the request session = request._get_session(URI) session.post.assert_called_once_with(URI, data=b'request', timeout=60) # Ensure the adapter parameters match those we specified check_adapters_mounted(session) adapter = session.get_adapter(URI) assert isinstance(adapter, HTTPAdapter) assert adapter._pool_connections == 100 assert adapter._pool_maxsize == 100
def __init__(self, base_url: str, *args, **kwargs): """ Creates a new HTTP session instance. Args: base_url: The base URL for this HTTP session *args: The optional positional arguments **kwargs: The optional keyword arguments """ super().__init__(*args, **kwargs) self._base_url = base_url # Custom headers self.headers.update({ 'User-Agent': 'geodatabr/{version} ({url})'.format(version=__meta__.__version__, url=__meta__.__url__), }) # Automatic retries for protocol in ('http://', 'https://'): self.mount( protocol, adapters.HTTPAdapter(max_retries=retry.Retry( total=HTTP_MAX_RETRIES, backoff_factor=HTTP_BACKOFF_FACTOR, status_forcelist=HTTP_RETRY_STATUSES)))
def __init__(self, auth_callback=None, session=None, client_id=None, client_secret=None, access_token=None, access_token_cache_file=None): self._res = {} self.auth_callback = auth_callback self.pin = None self._access_token_cache_file = access_token_cache_file self._client_id = client_id self._client_secret = client_secret self._access_token = access_token if (access_token_cache_file is not None and access_token is None and os.path.exists(access_token_cache_file)): with open(access_token_cache_file, 'r') as f: self._res = json.load(f) self._callback(self._res) if session is not None: session = weakref.ref(session) self._session = session self._adapter = adapters.HTTPAdapter()
def __init__(self, config, logger): self._logutils = LogUtils(logger) self._config = config self._url = config.get_service_url() self._request_uri = self._url.geturl() + HttpConstants.NOSQL_DATA_PATH self._pool_connections = config.get_pool_connections() self._pool_maxsize = config.get_pool_maxsize() self._max_request_id = 1 self._proxy_host = config.get_proxy_host() self._proxy_port = config.get_proxy_port() self._proxy_username = config.get_proxy_username() self._proxy_password = config.get_proxy_password() self._retry_handler = config.get_retry_handler() if self._retry_handler is None: self._retry_handler = DefaultRetryHandler() self._sec_info_timeout = config.get_sec_info_timeout() self._shut_down = False self._user_agent = self._make_user_agent() self._auth_provider = config.get_authorization_provider() if self._auth_provider is None: raise IllegalArgumentException( 'Must configure AuthorizationProvider.') self._sess = Session() adapter = adapters.HTTPAdapter(pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, max_retries=5, pool_block=True) self._sess.mount(self._url.scheme + '://', adapter) if self._proxy_host is not None: self._check_and_set_proxy(self._sess)
def new_connection(self, cluster_api, provider): config = cluster_api.nsxlib_config session = TimeoutSession(config.http_timeout, config.http_read_timeout) if config.client_cert_provider: session.cert_provider = config.client_cert_provider else: session.auth = (provider.username, provider.password) # NSX v3 doesn't use redirects session.max_redirects = 0 session.verify = not config.insecure if session.verify and provider.ca_file: # verify using the said ca bundle path session.verify = provider.ca_file # we are pooling with eventlet in the cluster class adapter = adapters.HTTPAdapter(pool_connections=1, pool_maxsize=1, max_retries=config.retries, pool_block=False) session.mount('http://', adapter) session.mount('https://', adapter) self.get_default_headers(session, provider, config.allow_overwrite_header) return session
def _make_adapter_with_retries() -> adapters.HTTPAdapter: retry = urllib3.Retry( connect=5, total=10, method_whitelist=["GET"], status_forcelist=[500, 501, 502, 503], ) return adapters.HTTPAdapter(max_retries=retry)
def __init__(self, *args, **kwargs): super(Consumer, self).__init__(*args, **kwargs) if not self.type: self.type = self.TYPE # attach session self.session = requests.Session() adapter = adapters.HTTPAdapter(max_retries=3) self.session.mount('http://', adapter)
def query_api(full_url): session = requests.Session() for scheme in ('http://', 'https://'): session.mount( scheme, adapters.HTTPAdapter(max_retries=retry.Retry(connect=5), )) r_s = session.get(full_url) return r_s
def make_session(http_max_pool_size, num_retries): session = grequests.Session() adapter = adapters.HTTPAdapter(max_retries=num_retries, pool_maxsize=http_max_pool_size, pool_block=True) session.mount('http://', adapter) session.mount('https://', adapter) return session
def _adapter_pool(url): """ Store and reuse HTTP adapters per Service URL. """ if url not in _ADAPTERS: _ADAPTERS[url] = adapters.HTTPAdapter() return _ADAPTERS[url]
def _make_adapter_with_retries(): retry = util.Retry( connect=5, total=10, method_whitelist=['GET'], status_forcelist=[500, 501, 502, 503], ) return adapters.HTTPAdapter(max_retries=retry)
def get(self, url): """ Store and reuse HTTP adapters per Service URL. """ if url not in self._adapters: self._adapters[url] = adapters.HTTPAdapter() return self._adapters[url]
def create(self): """ Create the requests.Session to communicate with the RESTful API. :return: :const:`None` :rtype: NoneType """ if self._session: self.close() if not self._session: self._session = requests.Session() self._session.mount('http://', ra.HTTPAdapter(max_retries=self._max_retries)) self._session.mount('https://', ra.HTTPAdapter(max_retries=self._max_retries)) msg = u'Created internal requests Session instance {0:#0x}' log_with_debug_info(logging.DEBUG, msg.format(id(self._session)))
def start_http_session() -> requests.Session: """Открытие клиентского соединение с интернетом.""" # logging.info("Открывается сессия для обновления данных по интернет") session = requests.Session() adapter = adapters.HTTPAdapter(pool_maxsize=HTTPS_MAX_POOL_SIZE, max_retries=MAX_RETRIES, pool_block=True) session.mount("https://", adapter) return session
def adapter(self): retry = util.Retry( connect=5, total=10, method_whitelist=["GET"], status_forcelist=[500, 501, 502, 503], ) return adapters.HTTPAdapter(max_retries=retry)
def upload_ptpb(rootpath, filename, notify_bool=True, name="File"): """Upload file to ptpb.pw and notify via notify-send. Keyword arguments: rootpath -- root path for file. filename -- name of file. notify_bool -- boolean to send notification (default True) name -- name used to generate nicer notifications (default File). """ from requests import Session, adapters, exceptions filepath = os.path.join(rootpath, filename) if not os.path.isfile(filepath): print("File not found in '" + filepath + "'.") return (False) url = "https://ptpb.pw" files = {'c': open(filepath, 'rb')} opts = {'sunset': '432000'} s = Session() a = adapters.HTTPAdapter(max_retries=3) s.mount('https://', a) try: rh = s.get(url) except exceptions.RequestException as e: print("Error: Failed to connect to " + url) notify( name + ":", name + " '" + filename + "'could not be uploaded." + " Failed to connect to " + url) return False if not (rh.status_code == 200): print("Error: Failed to connect to " + url) print(rh.status_code) if notify_bool: notify( name + ":", name + " " + filename + "could not be uploaded." + " Failed to connect to " + url) return False r = s.post(url + "/?u=1", files=files, data=opts) if (r.status_code == 200): pasteurl = r.headers['Location'] if notify_bool: notify(name + ":", name + " " + filename + " uploaded to:" + " " + pasteurl) else: if notify_bool: notify( name + ":", name + " " + filename + "could not be uploaded." + " Status code: " + r.status_code) print("Error: Upload was unsuccessful.") print(r.status_code) return False return (pasteurl)
def _init_values(self, target): self._folder = target # if target and not self.sin_sub: # self._folder = self._create_folders(target) self._emitidas = False self._current_year = datetime.datetime.now().year self._session = Session() a = adapters.HTTPAdapter(pool_connections=512, pool_maxsize=512, max_retries=5) self._session.mount('https://', a) return
def initclass(cls): if not cls._initialized: cls.logger = logging.getLogger(cls.__module__) cls._session = Session() adapter = adapters.HTTPAdapter(max_retries=5) cls._session.mount('http://', adapter) cls._session.mount('https://', adapter) cls._reqlock = Lock() cls._initialized = True cls.logger.debug('Initializing ' + cls.__name__)
def __init__(self, server_http_url="", *args, **kwargs): self.server_http_url = server_http_url self.session = requests.session() self.session.headers = {"Accept": "application/json", "Content-type": "application/json"} self.session.verify = False # Increase the number of pool connections so we can test large numbers # of connections to the api. adapter = adapters.HTTPAdapter(pool_connections=2000, pool_maxsize=2000) self.session.mount("http://", adapter)
def __init__(self, time_out=30, pool_num=10, pool_max_size=50, max_retries=3): super(SyncClientSession, self).__init__() self._time_out = time_out self._pool_num = pool_num self._pool_max_size = pool_max_size self._max_retries = max_retries self.mount( "http://", adapters.HTTPAdapter(pool_connections=self._pool_num, pool_maxsize=self._pool_max_size, max_retries=self._max_retries)) self.mount( "https://", adapters.HTTPAdapter(pool_connections=self._pool_num, pool_maxsize=self._pool_max_size, max_retries=self._max_retries))
def _build_adapters(self): """Parse adapter_settings to build adapters and mount within this session If retry settings are included, a retry object is built for the adapter. """ for prefix, settings in self.adapter_settings.items(): adapter = adapters.HTTPAdapter() if 'adapter' in settings: if 'retry' in settings: adapter = adapters.HTTPAdapter( max_retries=Refry(**settings['retry']), **settings['adapter'] ) else: adapter = adapters.HTTPAdapter(**settings['adapter']) adapter.config.update(settings.get('adapter_config', {})) self.mount(prefix, adapter)