def get_connect(self, host=None, port=80):
size = self.pool.qsize()
if size:
now = time.time()
while True:
try:
conn = self.pool.get_nowait()
if self.verify_connect(conn, now):
return conn
else:
size -= 1
if size <= 0:
break
except self.import_queue.Empty:
return None
except Exception as e:
logger.error("异常信息：%s", e)
size -= 1
if size <= 0:
break
try:
new_item = self.conn_factory(host, port, self.backend_mod)
new_item.connect()
return new_item
except Exception as e:
logger.error("创建连接异常：%s", e)
return None
def connect_all(self):
size = self.pool.qsize()
if size:
while True:
try:
size -= 1
conn = self.pool.get_nowait()
if conn.is_valid():
conn.connect()
self.pool.put_nowait(conn)
else:
conn.invalidate()
if size <= 0:
break
except self.import_queue.Full:
break
except self.import_queue.Empty:
break
except Exception as e:
logger.error("异常信息：%s", e)
for i in range(self.max_count - self.pool.qsize()):
new_connect = None
try:
new_connect = self.conn_factory(self.host, self.port, self.backend_mod, True)
self.pool.put_nowait(new_connect)
except self.import_queue.Full:
if new_connect:
new_connect.invalidate()
break
except Exception as e:
logger.error('新建连接异常：%s', e)
logger.info("与主机[%s]端口[%s]新建[%s]个连接", self.host, self.port, self.pool.qsize())
def get_socket_http_response(sock):
charset = 'utf-8'
_UNKNOWN = 'UNKNOWN'
http_response = None
# 接收html字节数据
r = client.HTTPResponse(sock)
try:
try:
r.begin()
except ConnectionError as ce:
logger.error('拉取数据异常：%s', ce)
will_close = r.will_close
http_response = HTTPResponse.from_httplib(r)
if will_close and will_close != _UNKNOWN:
# logger.debug('数据已接收，主机关闭了连接')
sock.close()
except Exception as e:
logger.error('数据接收异常：%s', e)
finally:
r.close()
# print('response：')
# print(response.decode(charset))
# 保持连接
if http_response is not None:
setattr(http_response, "body", http_response.data.decode(charset))
return http_response
else:
return None
def invalidate_all(self):
if self.pool.qsize():
while True:
try:
self.pool.get_nowait().invalidate()
except self.import_queue.Empty:
break
except Exception as e:
logger.error("异常信息：%s", e)
logger.info("与主机[%s]端口[%s]连接已释放", self.host, self.port)
def get_cookies_str(cookie):
try:
if isinstance(cookie, CookieJar):
cookie_array = []
for _cookie in iter(cookie):
cookie_array.append(f'{_cookie.name}={_cookie.value};')
return ''.join(cookie_array)
elif isinstance(cookie, dict):
return cookies.cookiejar_from_dict(cookie).__str__()
elif isinstance(cookie, str):
return cookie
else:
logger.warning('cookie类型不匹配，使用空值')
return ''
except Exception as e:
logger.error('cookie转换异常，信息：%s', e)
return ''
def verify_all(self):
active_count = 0
now = time.time()
for i in range(self.max_count):
conn = None
try:
conn = self.pool.get_nowait()
if self.verify_connect(conn, now):
if conn.is_connected():
active_count += 1
elif self.active_count > active_count:
# 根据active_count值保持活跃连接数
conn.connect()
active_count += 1
self.pool.put_nowait(conn)
except self.import_queue.Empty:
break
except self.import_queue.Full:
break
except Exception as e:
logger.error("异常信息：%s", e)
if conn:
conn.invalidate()
# 完成后需要保证队列中有max_count个连接，不够则创建
left_count = self.max_count - self.pool.qsize()
if active_count >= self.active_count:
for i in range(left_count):
try:
new_connect = self.conn_factory(self.host, self.port, self.backend_mod)
self.pool.put_nowait(new_connect)
except self.import_queue.Full:
break
except Exception as e:
logger.error('新建连接异常，host：%s，port：%s，异常：%s', self.host, self.port, e)
else:
left_active_count = self.active_count - active_count
left_static_count = left_count - left_active_count
# 剩余空间足够
if left_static_count >= 0:
for i in range(left_active_count):
try:
new_connect = self.conn_factory(self.host, self.port, self.backend_mod, True)
self.pool.put_nowait(new_connect)
except self.import_queue.Full:
break
except Exception as e:
logger.error('新建连接异常，host：%s，port：%s，异常：%s', self.host, self.port, e)
for i in range(left_static_count):
try:
new_connect = self.conn_factory(self.host, self.port, self.backend_mod)
self.pool.put_nowait(new_connect)
except self.import_queue.Full:
break
except Exception as e:
logger.error('新建连接异常，host：%s，port：%s，异常：%s', self.host, self.port, e)
def get_connect(self, host=None, port=80):
conn = None
pool = self.pool_manager.get_pool(host, port)
if pool:
tries = 0
while tries < self.retry_count:
try:
conn = pool.get_connect(host, port)
break
except Exception as e:
logger.error('获取连接异常，重试第：%s次，异常：%s', tries, e)
tries += 1
try:
yield conn
except Exception as e:
logger.error('连接异常：%s', e)
if conn:
conn.invalidate()
else:
self.pool_manager.put_connect(conn)
def __init__(self, conn_factory, backend_mod=None,
host=None, port=80, active_count=3, max_count=10):
self.conn_factory = conn_factory
self.host = host
self.port = port
self.active_count = active_count
self.max_count = max_count
self.backend_mod = backend_mod
self.import_queue = getattr(backend_mod, 'queue')
self.pool = self.import_queue.Queue(max_count)
for i in range(active_count):
try:
new_connect = conn_factory(host, port, backend_mod, True)
if new_connect.is_connected():
self.pool.put_nowait(new_connect)
except self.import_queue.Full:
logger.error("队列已满")
break
except Exception as e:
logger.error('新建连接异常，host：%s，port：%s，异常：%s', host, port, e)
static_count = max_count - active_count
if static_count > 0:
for i in range(static_count):
try:
new_connect = conn_factory(host, port, backend_mod)
self.pool.put_nowait(new_connect)
except self.import_queue.Full:
logger.error("队列已满")
break
except Exception as e:
logger.error('新建连接异常，host：%s，port：%s，异常：%s', host, port, e)
self.sem = self.backend_mod.Semaphore(1)
def handle_exception(self, exception):
logger.error('连接异常：%s', exception)