def __init__(self, args):
self.debug = args.verbose
self.download_path = args.download_path
self.timeout = args.tester_timeout
self.max_concurrency = args.tester_max_concurrency
self.disable_anonymity = args.tester_disable_anonymity
self.notice_interval = args.tester_notice_interval
self.pogo_version = args.tester_pogo_version
self.scan_interval = args.proxy_scan_interval
self.ignore_country = args.proxy_ignore_country
self.proxy_judge = args.proxy_judge
self.local_ip = args.local_ip
self.ip2location = IP2LocationDatabase(args)
self.running = Event()
self.test_queue = Queue()
self.test_hashes = []
self.proxy_updates_lock = Lock()
self.proxy_updates = {}
self.stats = {'valid': 0, 'fail': 0, 'total_valid': 0, 'total_fail': 0}
urllib3.disable_warnings()
self.retries = urllib3.Retry(total=args.tester_retries,
backoff_factor=args.tester_backoff_factor,
status_forcelist=self.STATUS_FORCELIST)
def _build_session(self):
self._session = requests.Session()
retry = urllib3.Retry(total=self.retries,
connect=None,
read=False,
method_whitelist=frozenset(
['GET', 'POST', 'PUT', 'DELETE']),
status=self.status_retries,
backoff_factor=self.backoff_factor,
status_forcelist=self.status_forcelist)
adapter = requests.adapters.HTTPAdapter(max_retries=retry)
self._session.mount('http://', adapter)
self._session.mount('https://', adapter)
def _create_session(self):
"""
Create a new requests session and apply config values
"""
session = requests.Session()
retry_obj = urllib3.Retry(
# setting a total is necessary to cover SSL related errors
total=max(self._config['connect_retries'],
self._config['read_retries']),
connect=self._config['connect_retries'],
read=self._config['read_retries'],
method_whitelist=self._config['method_whitelist'],
redirect=self._config['redirect'])
session.mount('http://',
requests.adapters.HTTPAdapter(max_retries=retry_obj))
session.mount('https://',
requests.adapters.HTTPAdapter(max_retries=retry_obj))
return session
def __init__(self, *args, **kwargs):
retries = kwargs.pop("retries", 0)
cache = kwargs.pop("cache", None)
super(PypiSession, self).__init__(*args, **kwargs)
# Attach our User Agent to the request
self.headers["User-Agent"] = user_agent()
# Attach our Authentication handler to the session
self.auth = MultiDomainBasicAuth()
# Create our urllib3.Retry instance which will allow us to customize
# how we handle retries.
retries = urllib3.Retry(
# Set the total number of retries that a particular request can
# have.
total=retries,
# A 503 error from PyPI typically means that the Fastly -> Origin
# connection got interupted in some way. A 503 error in general
# is typically considered a transient error so we'll go ahead and
# retry it.
status_forcelist=[503],
# Add a small amount of back off between failed requests in
# order to prevent hammering the service.
backoff_factor=0.25,
)
# We want to _only_ cache responses on securely fetched origins. We do
# this because we can't validate the response of an insecurely fetched
# origin, and we don't want someone to be able to poison the cache and
# require manual eviction from the cache to fix it.
if cache:
secure_adapter = CacheControlAdapter(
cache=SafeFileCache(cache, use_dir_lock=True),
max_retries=retries,
)
else:
secure_adapter = HTTPAdapter(max_retries=retries)
self.mount("https://", secure_adapter)
self.mount("file://", LocalFSAdapter())
def make_request(self, method, url, signature, **kwargs):
"""Make a tracked request using the requests library and class stats client
Note that because FrySession is intended to be used across many dependencies, any session cookies are cleared
before making a request so as not to change the response in an unintended way. Cookies can still be passed via
the `cookie` kwarg.
Args:
method: HTTP method to use for request
url: url to make the request against
signature: Stats signature for this request (ex: {service}.{endpoint})
**kwargs: Any requests (library) kwargs to be passed along (i.e. data, headers, etc)
Returns:
requests.Response
"""
request_retries = 0
request_adapter = self.get_adapter(url)
adapter_retries = getattr(request_adapter, 'max_retries', urllib3.Retry(0)).total
timeout = getattr(request_adapter, 'config', {}).get('timeout', DEFAULT_TIMEOUT)
self.cookies.clear()
try:
response = self._perform_timed_request(method, url, timeout, signature, **kwargs)
self._track_status_code(signature, response.status_code)
request_retries = adapter_retries - response.raw.retries.total
return response
except Exception as ex:
# assume that all retries were exercised on a raised ConnectionError or Timeout
if isinstance(ex, (requests.exceptions.ConnectionError, requests.exceptions.Timeout)):
request_retries = adapter_retries
# Track the error as a 500 in the status code stats
self._track_status_code(signature, 500)
self._track_error(signature, ex)
raise
finally:
self._track_retries(signature, request_retries)
This is just for internal licenseware use:
- AUTH_SERVICE_MACHINES_URL_PATH (route auth to be used between services)
"""
import os
import requests
from requests.adapters import HTTPAdapter
from requests.packages import urllib3
from datetime import datetime, timedelta
from functools import wraps
from licenseware.utils.log_config import log
retry_strategy = urllib3.Retry(
total=3
)
adapter = HTTPAdapter(max_retries=retry_strategy)
http = requests.Session()
http.mount("https://", adapter)
http.mount("http://", adapter)
class Authenticator:
def __init__(self):
self.email = os.getenv("LWARE_IDENTITY_USER")
self.password = os.getenv("LWARE_IDENTITY_PASSWORD")
self.auth_url = os.getenv("AUTH_SERVICE_URL")
class TimeoutHTTPAdapter(HTTPAdapter):
def __init__(self, *args, **kwargs):
self.timeout = DEFAULT_TIMEOUT
if "timeout" in kwargs:
self.timeout = kwargs["timeout"]
del kwargs["timeout"]
super(TimeoutHTTPAdapter, self).__init__(*args, **kwargs)
def send(self, request, **kwargs):
timeout = kwargs.get("timeout")
if timeout is None:
kwargs["timeout"] = self.timeout
return super(TimeoutHTTPAdapter, self).send(request, **kwargs)
try:
retry_strategy = urllib3.Retry(allowed_methods=ALLOWED_METHODS,
**RETRY_KWARGS)
except TypeError:
# NOTE remove once urllib3 1.26 is the minimum version
retry_strategy = urllib3.Retry(method_whitelist=ALLOWED_METHODS,
**RETRY_KWARGS)
adapter = TimeoutHTTPAdapter(max_retries=retry_strategy)
session = requests.Session()
session.mount("https://", adapter)
session.mount("http://", adapter)
