def get_relist(self, id, Api): # GET /t/re_list i = 0 pt = 0 sw = 0 tid = 0 r_list = list() # 本来是想抓取所有的转发信息,但是动辄几千条,太浪费时间,改为了最多只抓取500条 while not i: t_relist = Api.get.t__re_list(format="json", flag=0, rootid=id, pageflag=1, pagetime=pt, reqnum=100, twitterid=tid) time.sleep(3.6) if t_relist.data is None: curTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) print "%s no relay list! %s" % (id, curTime) break r_list.extend(t_relist.data.info) sw += 1 if sw == 5: break i = t_relist.data.hasnext if not i: pt = t_relist.data.info[-1]["timestamp"] tid = t_relist.data.info[-1]["id"] if r_list: msc = MysqlControl() msc.insert_r_list(id, r_list)
def get_statuses(self, uid, Api): # GET /statuses/user_timeline # 每次抓取20条 w_list = list() t_list = Api.get.statuses__user_timeline(format="json", name=uid, pageflag=0, pagetime=0, reqnum=20, lastid=0, type=0, contenttype=0x80) time.sleep(3.6) if t_list.data is None: curTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) print "%s no statuses list! %s" % (uid, curTime) return [] else: w_list.extend(t_list.data.info) msc = MysqlControl() msc.insert_r_list(100, w_list, uid) return w_list
def insert_mutuals(self, mutual_name, Api): mysqlContol = MysqlControl() user_info = Api.get.user__other_info(format="json", name=mutual_name) mysqlContol.insert_user_info(user_info.data) time.sleep(3.6) n_list = self.get_mutual(mutual_name, Api) if n_list: mysqlContol.insert_f_relation(mutual_name, n_list) for name in n_list: user_info = Api.get.user__other_info(format="json", name=name) mysqlContol.insert_user_info(user_info.data) time.sleep(3.6) d_relation = self.get_relations(n_list, Api) if d_relation: for x in d_relation.keys(): y = d_relation[x] mysqlContol.insert_s_relation(mutual_name, x, y) curTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) print "Completed the tasks of input %s's mutual info! %s" % ( mutual_name, curTime)
def insert_mutuals(self, mutual_name, Api): mysqlContol = MysqlControl() user_info = Api.get.user__other_info(format="json", name=mutual_name) mysqlContol.insert_user_info(user_info.data) time.sleep(3.6) n_list = self.get_mutual(mutual_name, Api) if n_list: mysqlContol.insert_f_relation(mutual_name, n_list) for name in n_list: user_info = Api.get.user__other_info(format="json", name=name) mysqlContol.insert_user_info(user_info.data) time.sleep(3.6) d_relation = self.get_relations(n_list, Api) if d_relation: for x in d_relation.keys(): y = d_relation[x] mysqlContol.insert_s_relation(mutual_name, x, y) curTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) print "Completed the tasks of input %s's mutual info! %s" % (mutual_name, curTime)