def _get_page_sy(http_task, cookie):
"""以同步方式获取网页内容
Args:
task:HttpTask, 任务描述
cookie: str, cookie
Returns:
resp: HTTPResponse, 结果
Raises:
error:GetPageError 错误
"""
headers = {"User-Agent": DEFAULT_USER_AGENT,
"Accept-Encoding": DEFAULT_ACCEPT_ENCODING,
"Accept": DEFAULT_ACCEPT}
if cookie:
headers.update({"Cookie": cookie})
if http_task.request.headers:
http_task.request.headers.update(headers)
else:
http_task.request.headers = headers
# dns resovler
if http_task.dns_need:
try:
addr = DNSResolver.instance().resolve(http_task.request.url)
except ResolveError, e:
logger.warn("dns %s error:%s" % (http_task.request.url, e))
else:
http_task.request.url = addr
def resovle_dns_for_request(http_request):
"""resolve dns for request
Args:
http_request:HttpRequest, request
"""
try:
ip_addr = DNSResolver.instance().resolve(http_request.url)
except ResolveError, e:
logger.warn("dns error:%s, error:%s" % (http_request.host, e))
