def get_html(session, url, delay_time=0, headers=None):
time.sleep(delay_time)
try:
if USE_PROXY:
proxy = Proxys.select().where(Proxys.score > 0).order_by(
fn.Rand()).limit(1)
if proxy:
proxy_ip = "http://{0}:{1}".format(proxy[0].ip, proxy[0].port)
response = session.get(url,
headers=headers,
proxies={"https": proxy_ip})
else:
response = session.get(url, headers=headers)
else:
response = session.get(url, headers=headers)
response.encoding = "gbk"
except Exception as e:
raise RuntimeError(f"{ERR_PREFIX} get_html({url}) fail, error:{e}")
return response.text
async def async_get_html(url, delay_time=0, headers=None):
await asyncio.sleep(delay_time)
try:
if USE_PROXY:
proxy = await objects.execute(Proxys.select().where(
Proxys.score > 0).order_by(fn.Rand()).limit(1))
if proxy:
response = await http_client.fetch(url,
headers=headers,
proxy_host=proxy[0].ip,
proxy_port=proxy[0].port)
else:
response = await http_client.fetch(url, headers=headers)
else:
response = await http_client.fetch(url, headers=headers)
html = response.body.decode("gbk")
except Exception as e:
raise RuntimeError(
f"{ERR_PREFIX}async_get_html({url}) fail, error:{e}")
return html