def _get_page_sy(http_task, cookie): """以同步方式获取网页内容 Args: task:HttpTask, 任务描述 cookie: str, cookie Returns: resp: HTTPResponse, 结果 Raises: error:GetPageError 错误 """ headers = {"User-Agent": DEFAULT_USER_AGENT, "Accept-Encoding": DEFAULT_ACCEPT_ENCODING, "Accept": DEFAULT_ACCEPT} if cookie: headers.update({"Cookie": cookie}) if http_task.request.headers: http_task.request.headers.update(headers) else: http_task.request.headers = headers # dns resovler if http_task.dns_need: try: addr = DNSResolver.instance().resolve(http_task.request.url) except ResolveError, e: logger.warn("dns %s error:%s" % (http_task.request.url, e)) else: http_task.request.url = addr
def resovle_dns_for_request(http_request): """resolve dns for request Args: http_request:HttpRequest, request """ try: ip_addr = DNSResolver.instance().resolve(http_request.url) except ResolveError, e: logger.warn("dns error:%s, error:%s" % (http_request.host, e))