def getNewWechatExtInfo(): ''' modify by bigzhu at 15/09/13 17:25:43 为了用订阅号给指定用户发消息,改用WechatExt ''' wechat_ext = WechatExt(user_name, password, appid=appid ) token_cookies = wechat_ext.get_token_cookies() pg.insertOrUpdate(pg, 'wechat_dead_line', token_cookies, where='1=1') return wechat_ext
def saveUserCheckNew(blogs): ''' create by bigzhu at 15/09/05 11:57:11 ''' user = blogs['blog'] user_name = user['name'] where = "name='%s'" % user_name result = list(pg.select('tumblr_user', where=where)) if result: if result[0].updated == user['updated']: # 如果没有更新过,就不用继续了 print user_name, ' no update' # return pass pg.insertOrUpdate(pg, 'tumblr_user', user, where) blogs = blogs['posts'] saveBlogs(user_name, blogs, offset=20)
def getUser(user_name, always_check=False): ''' modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找 ''' users = list( pg.select('instagram_user', where="lower(username)=lower('%s')" % user_name)) if users and not always_check: return users[0] else: user = None try: for this_user in api.user_search(user_name, 10): if this_user.username.lower() == user_name.lower(): user = this_user break except instagram.bind.InstagramClientError: print public_bz.getExpInfoAll() return # 如果没有这个用户 if user is None: public_db.delNoName('instagram', user_name) return try: user = api.user(user.id) except (instagram.bind.InstagramAPIError, instagram.bind.InstagramClientError): # 通常是没有访问权限 print public_bz.getExpInfoAll() public_db.delNoName('instagram', user_name) return db_user = storage() db_user.id_str = user.id db_user.username = user.username db_user.full_name = user.full_name db_user.profile_picture = user.profile_picture db_user.bio = user.bio db_user.website = user.website db_user.counts = json.dumps(user.counts) #pg.insert('instagram_user', **db_user) pg.insertOrUpdate(pg, 'instagram_user', db_user, "id_str='%s'" % db_user.id_str) #db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id) return getUser(user_name)
def getUser(user_name, always_check=False): ''' modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找 ''' users = list(pg.select('instagram_user', where="lower(username)=lower('%s')" % user_name)) if users and not always_check: return users[0] else: user = None try: for this_user in api.user_search(user_name, 10): if this_user.username.lower() == user_name.lower(): user = this_user break except instagram.bind.InstagramClientError: print public_bz.getExpInfoAll() return # 如果没有这个用户 if user is None: public_db.delNoName('instagram', user_name) return try: user = api.user(user.id) except (instagram.bind.InstagramAPIError, instagram.bind.InstagramClientError): # 通常是没有访问权限 print public_bz.getExpInfoAll() public_db.delNoName('instagram', user_name) return db_user = storage() db_user.id_str = user.id db_user.username = user.username db_user.full_name = user.full_name db_user.profile_picture = user.profile_picture db_user.bio = user.bio db_user.website = user.website db_user.counts = json.dumps(user.counts) #pg.insert('instagram_user', **db_user) pg.insertOrUpdate(pg, 'instagram_user', db_user, "id_str='%s'" % db_user.id_str) #db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id) return getUser(user_name)
def saveTwitter(tweet): ''' create by bigzhu at 15/07/10 14:39:48 保存twitter ''' del tweet.user._json #del tweet.user._api tweet.user.entities = json.dumps(tweet.user.entities) del tweet.user.id pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.user), "id_str='%s'" % tweet.user.id_str) tweet.t_user_id = tweet.user.id_str screen_name = tweet.user.screen_name del tweet.user if hasattr(tweet, 'author'): #del tweet.author.id pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.author), "id_str='%s'" % tweet.author.id_str) tweet.t_author_id = tweet.author.id_str del tweet.author if hasattr(tweet, '_api'): del tweet._api if hasattr(tweet, '_json'): del tweet._json # twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range if hasattr(tweet, 'id'): del tweet.id if hasattr(tweet, 'entities'): tweet.entities = json.dumps(tweet.entities) if hasattr(tweet, 'geo'): tweet.geo = json.dumps(tweet.geo) if hasattr(tweet, 'coordinates'): tweet.coordinates = json.dumps(tweet.coordinates) if hasattr(tweet, 'extended_entities'): tweet.extended_entities = json.dumps(tweet.extended_entities) if hasattr(tweet, 'scopes'): tweet.scopes = json.dumps(tweet.scopes) if hasattr(tweet, 'retweeted_status'): saveTwitter(tweet.retweeted_status) tweet.retweeted_status = tweet.retweeted_status.id_str if hasattr(tweet, 'quoted_status'): # print tweet.quoted_status del tweet.quoted_status # saveTwitter(tweet.quoted_status) # tweet.quoted_status = tweet.quoted_status.id_str # place 是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050> if hasattr(tweet, 'place'): del tweet.place # for k, v in vars(tweet).items(): # print '%s=%s' % (k, v) #return pg.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str) m = public_bz.storage() m.id_str = tweet.id_str m.m_type = 'twitter' m.m_user_id = tweet.t_user_id m.created_at = tweet.created_at if hasattr(tweet, 'extended_entities'): m.extended_entities = tweet.extended_entities m.content = None m.text = tweet.text m.href = 'https://twitter.com/'+screen_name+'/status/'+tweet.id_str return pg.insertIfNotExist(pg, 'm', m, "id_str='%s' and m_type='twitter'" % tweet.id_str)
def saveTwitter(tweet): ''' create by bigzhu at 15/07/10 14:39:48 保存twitter ''' del tweet.user._json #del tweet.user._api tweet.user.entities = json.dumps(tweet.user.entities) del tweet.user.id pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.user), "id_str='%s'" % tweet.user.id_str) tweet.t_user_id = tweet.user.id_str screen_name = tweet.user.screen_name del tweet.user if hasattr(tweet, 'author'): #del tweet.author.id pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.author), "id_str='%s'" % tweet.author.id_str) tweet.t_author_id = tweet.author.id_str del tweet.author if hasattr(tweet, '_api'): del tweet._api if hasattr(tweet, '_json'): del tweet._json # twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range if hasattr(tweet, 'id'): del tweet.id if hasattr(tweet, 'entities'): tweet.entities = json.dumps(tweet.entities) if hasattr(tweet, 'geo'): tweet.geo = json.dumps(tweet.geo) if hasattr(tweet, 'coordinates'): tweet.coordinates = json.dumps(tweet.coordinates) if hasattr(tweet, 'extended_entities'): tweet.extended_entities = json.dumps(tweet.extended_entities) if hasattr(tweet, 'scopes'): tweet.scopes = json.dumps(tweet.scopes) if hasattr(tweet, 'retweeted_status'): saveTwitter(tweet.retweeted_status) tweet.retweeted_status = tweet.retweeted_status.id_str if hasattr(tweet, 'quoted_status'): # print tweet.quoted_status del tweet.quoted_status # saveTwitter(tweet.quoted_status) # tweet.quoted_status = tweet.quoted_status.id_str # place 是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050> if hasattr(tweet, 'place'): del tweet.place # for k, v in vars(tweet).items(): # print '%s=%s' % (k, v) #return pg.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str) m = public_bz.storage() m.id_str = tweet.id_str m.m_type = 'twitter' m.m_user_id = tweet.t_user_id m.created_at = tweet.created_at if hasattr(tweet, 'extended_entities'): m.extended_entities = tweet.extended_entities m.content = None m.text = tweet.text m.href = 'https://twitter.com/' + screen_name + '/status/' + tweet.id_str return pg.insertIfNotExist( pg, 'm', m, "id_str='%s' and m_type='twitter'" % tweet.id_str)