def send_get_request(url: str, params: dict = None, timeout: int = 3, **kwargs):
"""发起GET请求
Args:
url (str): 目标地址
params (dict, optional): 请求参数. Defaults to None.
timeout (int, optional): 超时时间. Defaults to 3.
Returns:
[type]: [description]
"""
try:
resp = requests.get(url, params, timeout=timeout, **kwargs)
except Exception as e:
resp = None
LOGGER.exception(f"请求出错 - {url} - {str(e)}")
return resp
def get_html_by_requests(url: str, params: dict = None, timeout: int = 3, **kwargs):
"""发起GET请求，获取文本
Args:
url (str): 目标网页
params (dict, optional): 请求参数. Defaults to None.
timeout (int, optional): 超时时间. Defaults to 3.
"""
resp = send_get_request(url=url, params=params, timeout=timeout, **kwargs)
text = None
try:
content = resp.content
charset = cchardet.detect(content)
text = content.decode(charset["encoding"])
except Exception as e:
LOGGER.exception(f"请求内容提取出错 - {url} - {str(e)}")
return text