def process_response(self, request, response, spider):
print('--------------------------中间件启动response---------------6-------')
if response.status in [301, 302, 300, 303]:
try:
redirect_url = response.headers['location']
if 'login.weibo' in redirect_url or 'login.sina' in redirect_url:
logger.warning('One Cookie need to updating..')
updateCookie(request.meta['accountText'], self.rconn,
spider.name)
elif 'weibo.cn/security' in redirect_url: #账户被限制
logger.warning('One Account is locked! Remove it')
removeCookie(request.meta['accountText'], self.rconn,
spider.name)
elif 'weibo.cn/pub' in redirect_url:
logger.warning(
'Redirect to "http://weibo.cn/pub"!(Account:%s)' %
request.meta['accountText'].split('--')[0])
reason = response_status_message(reponse.status)
return self._retry(request, reason, spider) or response
except Exception as e:
raise IgnoreRequest
elif response.status in [403, 414]:
logger.error('%s! stopping..' % response.status)
os.system('pause')
else:
return response
def process_response(self, request, response, spider):
# 根据 HTTP标准 ，返回值为200-300之间的值为成功的resonse
if response.status in [300, 301, 302, 303]:
try:
redirect_url = response.headers["location"]
if "login.weibo" in redirect_url or "login.sina" in redirect_url: # Cookie失效
strs = u"发现一个Cookie失效，需要更新........." + '\n'
print strs
logger.warning(u"发现一个Cookie失效，需要更新........." + '\n')
updateCookie(request.meta['accountText'], self.rconn,
spider.name)
elif "weibo.cn/security" in redirect_url: # 账号被限
strs = u"发现一个账号被限制，需要删除........." + '\n'
print strs
logger.warning(u"发现一个账号被限制，需要删除.........")
removeCookie(request.meta["accountText"], self.rconn,
spider.name)
elif "weibo.cn/pub" in redirect_url:
strs = u"重定向'http://weibo.cn/pub'!( Account:%s )" % request.meta[
"accountText"].split("--")[0] + '\n'
print strs
logger.warning(u"重定向'http://weibo.cn/pub'!( Account:%s )" %
request.meta["accountText"].split("--")[0])
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response # 重试
except Exception, e:
raise IgnoreRequest
def process_response(self, request, response, spider):
if response.status in [300, 301, 302, 303, 504]:
try:
redirect_url = response.headers["location"]
if "login.weibo" in redirect_url or "login.sina" in redirect_url: # Cookie失效
print("One Cookie need to be updating...")
updateCookie(request.cookies)
except Exception, e:
raise IgnoreRequest
def process_response(self, request, response, spider):
if response.status in [300, 301, 302, 303]:
try:
redirect_url = response.headers["location"]
if "login.weibo" in redirect_url or "login.sina" in redirect_url:: # Cookie失效
logger.warning("One Cookie need to be updating...")
updateCookie(request.meta['accountText'], self.rconn, spider.name)
elif "weibo.cn/security" in redirect_url: # 账号被限
logger.warning("One Account is locked! Remove it!")
removeCookie(request.meta["accountText"], self.rconn, spider.name)
elif "weibo.cn/pub" in redirect_url:
logger.warning(
"Redirect to 'http://weibo.cn/pub'!( Account:%s )" % request.meta["accountText"].split("--")[0])
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response # 重试
except Exception, e:
raise IgnoreRequest
def process_response(self, request, response, spider):
if response.status in [300, 301, 302, 303]:
try:
redirect_url = response.headers["location"]
if "login.weibo" in redirect_url or "login.sina" in redirect_url: # Cookie失效
logger.warning("One Cookie need to be updating...")
updateCookie(request.meta['accountText'], self.rconn, spider.name)
elif "weibo.cn/security" in redirect_url: # 账号被限
logger.warning("One Account is locked! Remove it!")
removeCookie(request.meta["accountText"], self.rconn, spider.name)
elif "weibo.cn/pub" in redirect_url:
logger.warning(
"Redirect to 'http://weibo.cn/pub'!( Account:%s )" % request.meta["accountText"].split("--")[0])
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response # 重试
except Exception, e:
raise IgnoreRequest
def process_response(self, request, response, spider):
prefix = BaseHelper.get_cookie_key_prefix(spider)
if response.status in [300, 301, 302, 303]:
try:
redirect_url = response.headers["location"]
if "login.qq" in redirect_url or "login.qq" in redirect_url: # Cookie失效
spider.logger.warning("One Cookie need to be updating...")
updateCookie(request.meta['accountText'], self.rconn,
spider)
elif "qq.cn/security" in redirect_url: # 账号被限
spider.logger.warning("One Account is locked! Remove it!")
removeCookie(request.meta["accountText"], self.rconn,
spider)
elif "qq.cn/pub" in redirect_url:
spider.logger.warning(
"Redirect to 'http://qq.com'!( Account:%s )" %
request.meta["accountText"].split("--")[0])
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response # 重试
except Exception, e:
raise IgnoreRequest
elif "qq.cn/pub" in redirect_url:
spider.logger.warning(
"Redirect to 'http://qq.com'!( Account:%s )" %
request.meta["accountText"].split("--")[0])
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response # 重试
except Exception, e:
raise IgnoreRequest
elif response.status in [403, 414]:
spider.logger.error("%s! Stopping..." % response.status)
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response
# os.system("pause")
elif u'登录态失效，请重新登录' in response.body or u'请先登录' in response.body:
spider.logger.warning("One Cookie need to be updating...")
updateCookie(request.meta['accountText'], self.rconn,
spider)
reason = response_status_message(response.status)
return self._retry(request, reason, spider) or response
else:
return response
@classmethod
def _replace_token(cls, url, cookies, spider):
if "GTK" in url:
p_skey = cookies.get("p_skey")
skey = cookies.get("skey")
rv2 = cookies.get("rv2")
spider.logger \
.debug("p_skey={}, skey={}, rv2={}".format(p_skey, skey, rv2))
str_key = p_skey or skey or rv2
g_tk = cls._gen_token(str_key)
