def process_response(self, request, response, spider): # 根据 HTTP标准 ,返回值为200-300之间的值为成功的resonse if response.status in [300, 301, 302, 303]: try: redirect_url = response.headers["location"] if "login.weibo" in redirect_url or "login.sina" in redirect_url: # Cookie失效 strs = u"发现一个Cookie失效,需要更新........." + '\n' print strs logger.warning(u"发现一个Cookie失效,需要更新........." + '\n') updateCookie(request.meta['accountText'], self.rconn, spider.name) elif "weibo.cn/security" in redirect_url: # 账号被限 strs = u"发现一个账号被限制,需要删除........." + '\n' print strs logger.warning(u"发现一个账号被限制,需要删除.........") removeCookie(request.meta["accountText"], self.rconn, spider.name) elif "weibo.cn/pub" in redirect_url: strs = u"重定向'http://weibo.cn/pub'!( Account:%s )" % request.meta[ "accountText"].split("--")[0] + '\n' print strs logger.warning(u"重定向'http://weibo.cn/pub'!( Account:%s )" % request.meta["accountText"].split("--")[0]) reason = response_status_message(response.status) return self._retry(request, reason, spider) or response # 重试 except Exception, e: raise IgnoreRequest
def process_response(self, request, response, spider): print('--------------------------中间件启动response---------------6-------') if response.status in [301, 302, 300, 303]: try: redirect_url = response.headers['location'] if 'login.weibo' in redirect_url or 'login.sina' in redirect_url: logger.warning('One Cookie need to updating..') updateCookie(request.meta['accountText'], self.rconn, spider.name) elif 'weibo.cn/security' in redirect_url: #账户被限制 logger.warning('One Account is locked! Remove it') removeCookie(request.meta['accountText'], self.rconn, spider.name) elif 'weibo.cn/pub' in redirect_url: logger.warning( 'Redirect to "http://weibo.cn/pub"!(Account:%s)' % request.meta['accountText'].split('--')[0]) reason = response_status_message(reponse.status) return self._retry(request, reason, spider) or response except Exception as e: raise IgnoreRequest elif response.status in [403, 414]: logger.error('%s! stopping..' % response.status) os.system('pause') else: return response
def process_response(self, request, response, spider): if response.status in [300, 301, 302, 303]: try: redirect_url = response.headers["location"] if "login.weibo" in redirect_url or "login.sina" in redirect_url:: # Cookie失效 logger.warning("One Cookie need to be updating...") updateCookie(request.meta['accountText'], self.rconn, spider.name) elif "weibo.cn/security" in redirect_url: # 账号被限 logger.warning("One Account is locked! Remove it!") removeCookie(request.meta["accountText"], self.rconn, spider.name) elif "weibo.cn/pub" in redirect_url: logger.warning( "Redirect to 'http://weibo.cn/pub'!( Account:%s )" % request.meta["accountText"].split("--")[0]) reason = response_status_message(response.status) return self._retry(request, reason, spider) or response # 重试 except Exception, e: raise IgnoreRequest
def process_response(self, request, response, spider): if response.status in [300, 301, 302, 303]: try: redirect_url = response.headers["location"] if "login.weibo" in redirect_url or "login.sina" in redirect_url: # Cookie失效 logger.warning("One Cookie need to be updating...") updateCookie(request.meta['accountText'], self.rconn, spider.name) elif "weibo.cn/security" in redirect_url: # 账号被限 logger.warning("One Account is locked! Remove it!") removeCookie(request.meta["accountText"], self.rconn, spider.name) elif "weibo.cn/pub" in redirect_url: logger.warning( "Redirect to 'http://weibo.cn/pub'!( Account:%s )" % request.meta["accountText"].split("--")[0]) reason = response_status_message(response.status) return self._retry(request, reason, spider) or response # 重试 except Exception, e: raise IgnoreRequest
def process_response(self, request, response, spider): prefix = BaseHelper.get_cookie_key_prefix(spider) if response.status in [300, 301, 302, 303]: try: redirect_url = response.headers["location"] if "login.qq" in redirect_url or "login.qq" in redirect_url: # Cookie失效 spider.logger.warning("One Cookie need to be updating...") updateCookie(request.meta['accountText'], self.rconn, spider) elif "qq.cn/security" in redirect_url: # 账号被限 spider.logger.warning("One Account is locked! Remove it!") removeCookie(request.meta["accountText"], self.rconn, spider) elif "qq.cn/pub" in redirect_url: spider.logger.warning( "Redirect to 'http://qq.com'!( Account:%s )" % request.meta["accountText"].split("--")[0]) reason = response_status_message(response.status) return self._retry(request, reason, spider) or response # 重试 except Exception, e: raise IgnoreRequest