def run(self): from requests.packages.urllib3.util.retry import Retry from requests.adapters import HTTPAdapter headers = {'User-Agent': 'yobot/1.0'} retries = Retry.from_int(0) s = requests.Session() s.mount('http://', HTTPAdapter(max_retries=retries)) s.mount('https://', HTTPAdapter(max_retries=retries)) qDebug(str(self.urls).encode()) for url in self.urls: try: resp = s.get(url, headers=headers, timeout=1.2) qDebug('{}, {}'.format(resp.status_code, resp.encoding).encode()) self.resps[url] = resp except Exception as ex: qDebug(str(ex).encode()) self.resps[url] = None return self.seq
def __init__(self, pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES, pool_block=DEFAULT_POOLBLOCK): if max_retries == DEFAULT_RETRIES: self.max_retries = Retry(0, read=False) else: self.max_retries = Retry.from_int(max_retries) self.config = {} self.proxy_manager = {} super(HTTPAdapter, self).__init__() self._pool_connections = pool_connections self._pool_maxsize = pool_maxsize self._pool_block = pool_block self._pool_kw_lock = RLock() self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)