def _get_proxy_headers(self, url: str) -> dict:
"""获取代理访问使用的 Headers"""
headers = self.set_proxy_headers(url)
if not headers:
return {"User-Agent": get_random_ua()}
if "user-agent" not in (key.lower() for key in headers.keys()):
headers["User-Agent"] = get_random_ua()
return headers
def set_headers(kwargs: dict):
"""为请求设置 headers, 使用随机 User-Agent"""
kwargs.setdefault("timeout", ClientTimeout(total=30,
sock_connect=5)) # 连接超时
if "headers" not in kwargs: # 没有设置 Headers
kwargs["headers"] = {"User-Agent": get_random_ua()}
return
keys = [key.lower() for key in kwargs.get("headers")]
if "user-agent" not in keys: # 有 Header, 无 User-Agent
kwargs["headers"]["user-agent"] = get_random_ua()
def set_headers(self, url: str, kwargs: dict) -> str:
"""为请求设置 headers, 使用随机 User-Agent"""
kwargs.setdefault("timeout", ClientTimeout(total=30,
sock_connect=5)) # 连接超时
if "headers" not in kwargs: # 没有设置 Headers
kwargs["headers"] = {"User-Agent": get_random_ua()}
else:
keys = [key.lower() for key in kwargs.get("headers")]
if "user-agent" not in keys: # 有 Header, 无 User-Agent
kwargs["headers"]["user-agent"] = get_random_ua()
new_url, netloc = self._url_mapping(url)
if new_url != url: # 需要映射
kwargs["headers"]["host"] = netloc
return new_url