def get_response_content(self, url, headers = {}, data = None): ''' 获取响应数据 ''' content = '' try: # 关键的一步,加载模拟登录获取的cookie if os.path.exists(self.cookiefile): self.cj.revert(self.cookiefile, True, True) self.cookie_support = urllib2.HTTPCookieProcessor(self.cj) if self.proxy == '': self.opener = urllib2.build_opener(self.cookie_support, urllib2.HTTPHandler) else: self.opener = urllib2.build_opener(self.cookie_support, urllib2.ProxyHandler({'http': self.proxy})) urllib2.install_opener(self.opener) else: return '' req = self.pack_request(url = url, headers = headers, data = data) response = self.opener.open(req, timeout = 10) if response.info().get('Content-Encoding') == 'gzip': content = self.gzip_data(response.read()) else: content = response.read() except urllib2.HTTPError, e: logError(e) return e.code
def get_servertime(self): """ 模拟登录第一步,获取servertime、nonce等信息,用于登录时加密用户名、密码 """ url = 'http://login.sina.com.cn/sso/prelogin.php?entry=account&callback=sinaSSOController.preloginCallBack&su=&rsakt=mod&client=ssologin.js(v1.4.2)&_=%s' % self.__get_millitime( ) result = {} servertime = None nonce = None headers = self.__get_headers() headers['Host'] = 'login.sina.com.cn' headers['Accept'] = '*/*' headers['Referer'] = 'http://weibo.com/' del headers['Accept-encoding'] for i in range(3): req = self.pack_request(url, headers) data = urllib2.urlopen(req).read() p = re.compile('\((.*)\)') try: json_data = p.search(data).group(1) data = json.loads(json_data) result['servertime'] = str(data['servertime']) result['nonce'] = str(data['nonce']) result['rsakv'] = str(data['rsakv']) result['pubkey'] = str(data['pubkey']) self.pcid = str(data['pcid']) break except Exception, e: logError(e) msg = 'get severtime error!' logError(msg) continue
def get_response_content(self, url, headers={}, data=None): ''' 获取响应数据 ''' content = '' try: # 关键的一步,加载模拟登录获取的cookie if os.path.exists(self.cookiefile): self.cj.revert(self.cookiefile, True, True) self.cookie_support = urllib2.HTTPCookieProcessor(self.cj) if self.proxy == '': self.opener = urllib2.build_opener(self.cookie_support, urllib2.HTTPHandler) else: self.opener = urllib2.build_opener( self.cookie_support, urllib2.ProxyHandler({'http': self.proxy})) urllib2.install_opener(self.opener) else: return '' req = self.pack_request(url=url, headers=headers, data=data) response = self.opener.open(req, timeout=10) if response.info().get('Content-Encoding') == 'gzip': content = self.gzip_data(response.read()) else: content = response.read() except urllib2.HTTPError, e: logError(e) return e.code
def redo_login(self, login_url): ''' 第三步登录 ''' try: headers = self.__get_headers() headers[ 'Referer'] = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.2)' req = self.pack_request(login_url, headers) urllib2.urlopen(req) # 保存cookie!! self.cj.save(self.cookiefile, True, True) msg = 'login success' logInfo(msg) loginFalg = True except Exception, e: logError(e) s = sys.exc_info() msg = ('redo_login %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg) loginFalg = False
def get_servertime(self): """ 模拟登录第一步,获取servertime、nonce等信息,用于登录时加密用户名、密码 """ url = 'http://login.sina.com.cn/sso/prelogin.php?entry=account&callback=sinaSSOController.preloginCallBack&su=&rsakt=mod&client=ssologin.js(v1.4.2)&_=%s' % self.__get_millitime() result = {} servertime = None nonce = None headers = self.__get_headers() headers['Host'] = 'login.sina.com.cn' headers['Accept'] = '*/*' headers['Referer'] = 'http://weibo.com/' del headers['Accept-encoding'] for i in range(3): req = self.pack_request(url, headers) data = urllib2.urlopen(req).read() p = re.compile('\((.*)\)') try: json_data = p.search(data).group(1) data = json.loads(json_data) result['servertime'] = str(data['servertime']) result['nonce'] = str(data['nonce']) result['rsakv'] = str(data['rsakv']) result['pubkey'] = str(data['pubkey']) self.pcid = str(data['pcid']) break except Exception, e: logError(e) msg = 'get severtime error!' logError(msg) continue
def redo_login(self, login_url): ''' 第三步登录 ''' try: headers = self.__get_headers() headers['Referer'] = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.2)' req = self.pack_request(login_url, headers) urllib2.urlopen(req) # 保存cookie!! self.cj.save(self.cookiefile, True, True) msg = 'login success' logInfo(msg) loginFalg = True except Exception, e: logError(e) s = sys.exc_info() msg = ('redo_login %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg) loginFalg = False
def valid_cookie(self, html=''): ''' 验证本地cookie文件 ''' html = str(html) if not html: headers = self.__get_headers() # 测试李开复主页,判断cookie是否过期 html = self.get_response_content(url='http://weibo.com/kaifulee', headers=headers) if not html: msg = 'need relogin.' logError(msg) self.clear_cookiedat(self.cookiefile) return False html = str(html) html = html.replace('"', "'") if 'sinaSSOController' in html: p = re.compile('location\.replace\(\'(.*?)\'\)') try: login_url = p.search(html).group(1) headers = self.__get_headers() headers['Host'] = 'account.weibo.com' req = self.pack_request(url=login_url, headers=headers) result = urllib2.urlopen(req) self.cj.save(self.cookiefile, True, True) if result.info().get('Content-Encoding') == 'gzip': html = self.gzipData(result.read()) else: html = result.read() except Exception, e: logError(e) msg = 'relogin failure.' logError(msg) self.clear_cookiedat(self.cookiefile) return False
def valid_cookie(self, html = ''): ''' 验证本地cookie文件 ''' html = str(html) if not html: headers = self.__get_headers() # 测试李开复主页,判断cookie是否过期 html = self.get_response_content(url = 'http://weibo.com/kaifulee', headers = headers) if not html: msg = 'need relogin.' logError(msg) self.clear_cookiedat(self.cookiefile) return False html = str(html) html = html.replace('"', "'") if 'sinaSSOController' in html: p = re.compile('location\.replace\(\'(.*?)\'\)') try: login_url = p.search(html).group(1) headers = self.__get_headers() headers['Host'] = 'account.weibo.com' req = self.pack_request(url = login_url, headers = headers) result = urllib2.urlopen(req) self.cj.save(self.cookiefile, True, True) if result.info().get('Content-Encoding') == 'gzip': html = self.gzipData(result.read()) else: html = result.read() except Exception, e: logError(e) msg = 'relogin failure.' logError(msg) self.clear_cookiedat(self.cookiefile) return False
def run(self): # print到文件用 # reload(sys) # sys.setdefaultencoding('utf-8') try: searchResult = '' url = 'http://s.weibo.com/weibo/%s&xsort=time&scope=ori×cope=custom:%s:%s&page=%d' \ % (self.key, self.startTime, self.endTime, self.id) headers = { 'Host': 's.weibo.com', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:13.0) Gecko/20100101 Firefox/13.0.1', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3', 'Accept-Encoding': 'gzip, deflate', 'Connection': 'keep-alive', 'Referer': 'http://s.weibo.com', } content = self.sina.get_response_content(url, headers) if content == '': msg = u'%s failure:获取网页内容为空!' % self.id logError(msg) searchResult = 'error' # 其他类型错误处理 # 后续处理,提取微博信息 # result = open('result.html', 'w') # print >> result, content self.fetch(content) except Exception, e: logError(e) searchResult = 'error' s = sys.exc_info() msg = ('SearchWeiboThread run Error %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg)
def clear_cookiedat(self, datpath): try: os.remove(datpath) except Exception, e: logError(e)
def do_login(self, login_un, login_pw, door = ''): ''' 第二步登录 ''' loginFlag = False try: username = login_un pwd = login_pw url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.2)' # 构造POST体是关键! postdata = { 'entry': 'weibo', 'gateway': '1', 'from': '', 'savestate': '7', 'userticket': '1', 'pagerefer' : '', 'ssosimplelogin': '******', 'vsnf': '1', 'vsnval': '', 'service': 'miniblog', 'pwencode': 'rsa2', 'rsakv' : self.rsakv, 'encoding': 'UTF-8', 'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack', 'returntype': 'META', 'prelt' : '26', } postdata['servertime'] = self.servertime postdata['nonce'] = self.nonce postdata['su'] = self.get_user(username) postdata['sp'] = self.get_pwd(pwd, self.servertime, self.nonce).lower() # 当需要验证码登录的时候,后续优化 if door: postdata['pcid'] = self.pcid postdata['door'] = door.lower() headers = { 'Host': 'login.sina.com.cn', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:17.0) Gecko/20100101 Firefox/17.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-encoding': 'gzip, deflate', 'Accept-Language': 'zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3', 'Connection': 'keep-alive', 'Referer' : 'http://weibo.com/', 'Content-Type': 'application/x-www-form-urlencoded', } req = self.pack_request(url, headers, postdata) result = urllib2.urlopen(req) if result.info().get('Content-Encoding') == 'gzip': text = self.gzip_data(result.read()) else: text = result.read() return text except Exception, e: logError(e) s = sys.exc_info() msg = ('do_login: %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg) loginFlag = False
def login(self, login_un, login_pw): ''' 对外暴露的登录接口 ''' loginFalg = False try: try: # 步骤一,获取加密用servertime、nonce等信息 stObj = self.get_servertime() self.servertime = stObj.get('servertime') self.nonce = stObj.get('nonce') self.pubkey = stObj.get('pubkey') self.rsakv = stObj.get('rsakv') except Exception, e: logError(e) return False self.get_global_id() # 步骤二,加密密码登录 loginHtml = self.do_login(login_un, login_pw) loginHtml = loginHtml.replace('"', "'") try: p = re.compile('location\.replace\(\'(.*?)\'\)') login_url = p.search(loginHtml).group(1) if 'retcode=0' in loginHtml: # 步骤三,根据步骤二跳转地址,进一步登录,获取cookie信息 # 这一步成功才是真的成功!! return self.redo_login(login_url) if 'retcode=5' in loginHtml: logError('password or account error.') return False if 'retcode=4040' in loginHtml: logError('do login too much times.') return False # 需要验证码,悲剧,先报错吧!后续优化~ if 'retcode=4049' in login_url: logError('nead input verify code, return failure.') return False except Exception, e: logError(e) s = sys.exc_info() msg = ('do login %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg) loginFalg = False
def login(self, login_un, login_pw): ''' 对外暴露的登录接口 ''' loginFalg = False try: try: # 步骤一,获取加密用servertime、nonce等信息 stObj = self.get_servertime() self.servertime = stObj.get('servertime') self.nonce = stObj.get('nonce') self.pubkey = stObj.get('pubkey') self.rsakv = stObj.get('rsakv') except Exception, e: logError(e) return False self.get_global_id() # 步骤二,加密密码登录 loginHtml = self.do_login(login_un, login_pw) loginHtml = loginHtml.replace('"', "'") try: p = re.compile('location\.replace\(\'(.*?)\'\)') login_url = p.search(loginHtml).group(1) if 'retcode=0' in loginHtml: # 步骤三,根据步骤二跳转地址,进一步登录,获取cookie信息 # 这一步成功才是真的成功!! return self.redo_login(login_url) if 'retcode=5' in loginHtml: logError('password or account error.') return False if 'retcode=4040' in loginHtml: logError('do login too much times.') return False # 需要验证码,悲剧,先报错吧!后续优化~ if 'retcode=4049' in login_url: logError('nead input verify code, return failure.') return False except Exception, e: logError(e) s = sys.exc_info() msg = ('do login %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg) loginFalg = False
def clear_cookiedat(self, datpath): try: os.remove(datpath) except Exception, e: logError(e)
def do_login(self, login_un, login_pw, door=''): ''' 第二步登录 ''' loginFlag = False try: username = login_un pwd = login_pw url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.2)' # 构造POST体是关键! postdata = { 'entry': 'weibo', 'gateway': '1', 'from': '', 'savestate': '7', 'userticket': '1', 'pagerefer': '', 'ssosimplelogin': '******', 'vsnf': '1', 'vsnval': '', 'service': 'miniblog', 'pwencode': 'rsa2', 'rsakv': self.rsakv, 'encoding': 'UTF-8', 'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack', 'returntype': 'META', 'prelt': '26', } postdata['servertime'] = self.servertime postdata['nonce'] = self.nonce postdata['su'] = self.get_user(username) postdata['sp'] = self.get_pwd(pwd, self.servertime, self.nonce).lower() # 当需要验证码登录的时候,后续优化 if door: postdata['pcid'] = self.pcid postdata['door'] = door.lower() headers = { 'Host': 'login.sina.com.cn', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:17.0) Gecko/20100101 Firefox/17.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-encoding': 'gzip, deflate', 'Accept-Language': 'zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3', 'Connection': 'keep-alive', 'Referer': 'http://weibo.com/', 'Content-Type': 'application/x-www-form-urlencoded', } req = self.pack_request(url, headers, postdata) result = urllib2.urlopen(req) if result.info().get('Content-Encoding') == 'gzip': text = self.gzip_data(result.read()) else: text = result.read() return text except Exception, e: logError(e) s = sys.exc_info() msg = ('do_login: %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg) loginFlag = False
logError('do login too much times.') return False # 需要验证码,悲剧,先报错吧!后续优化~ if 'retcode=4049' in login_url: logError('nead input verify code, return failure.') return False except Exception, e: logError(e) s = sys.exc_info() msg = ('do login %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg) loginFalg = False except Exception, e: logError(e) s = sys.exc_info() msg = ('login: %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg) loginFalg = False return loginFalg def do_login(self, login_un, login_pw, door = ''): ''' 第二步登录 ''' loginFlag = False try:
return False # 需要验证码,悲剧,先报错吧!后续优化~ if 'retcode=4049' in login_url: logError('nead input verify code, return failure.') return False except Exception, e: logError(e) s = sys.exc_info() msg = ('do login %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg) loginFalg = False except Exception, e: logError(e) s = sys.exc_info() msg = ('login: %s happened on line %d' % (s[1], s[2].tb_lineno)) logError(msg) loginFalg = False return loginFalg def do_login(self, login_un, login_pw, door=''): ''' 第二步登录 ''' loginFlag = False try: username = login_un