def saveTwitter(tweet): ''' create by bigzhu at 15/07/10 14:39:48 保存twitter ''' if hasattr(tweet, 'user'): del tweet.user._json #del tweet.user._api tweet.user.entities = json.dumps(tweet.user.entities) del tweet.user.id db_bz.insertIfNotExist(pg, 'twitter_user', vars(tweet.user), "id_str='%s'" % tweet.user.id_str) tweet.t_user_id = tweet.user.id_str del tweet.user if hasattr(tweet, 'author'): #del tweet.author.id db_bz.insertIfNotExist(pg, 'twitter_user', vars(tweet.author), "id_str='%s'" % tweet.author.id_str) tweet.t_author_id = tweet.author.id_str del tweet.author if hasattr(tweet, '_api'): del tweet._api if hasattr(tweet, '_json'): del tweet._json # twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range if hasattr(tweet, 'id'): del tweet.id if hasattr(tweet, 'entities'): tweet.entities = json.dumps(tweet.entities) if hasattr(tweet, 'geo'): tweet.geo = json.dumps(tweet.geo) if hasattr(tweet, 'coordinates'): tweet.coordinates = json.dumps(tweet.coordinates) if hasattr(tweet, 'extended_entities'): tweet.extended_entities = json.dumps(tweet.extended_entities) if hasattr(tweet, 'retweeted_status'): saveTwitter(tweet.retweeted_status) tweet.retweeted_status = tweet.retweeted_status.id_str if hasattr(tweet, 'quoted_status'): # print tweet.quoted_status del tweet.quoted_status # saveTwitter(tweet.quoted_status) # tweet.quoted_status = tweet.quoted_status.id_str # place 是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050> if hasattr(tweet, 'place'): del tweet.place # for k, v in vars(tweet).items(): # print '%s=%s' % (k, v) return db_bz.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str)
def saveMessage(message): ''' create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里 ''' message.id_str = message.pop('id') content = storage() content.type = message.type content.repo = message.pop('repo') content.payload = message.pop('payload') message.content = json.dumps(content) if message.get('org'): message.org = json.dumps(message.org) # return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str) m = public_bz.storage() m.id_str = message.id_str m.m_type = 'github' m.m_user_id = message.actor m.created_at = message.created_at m.content = message.content m.text = None m.href = None return db_bz.insertIfNotExist( pg, 'm', m, "id_str='%s' and m_type='github'" % message.id_str)
def saveMessage(message): ''' create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里 ''' message.id_str = message.pop('id') content = storage() content.type = message.type content.repo = message.pop('repo') content.payload = message.pop('payload') message.content = json.dumps(content) if message.get('org'): message.org = json.dumps(message.org) # return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str) m = public_bz.storage() m.id_str = message.id_str m.m_type = 'github' m.m_user_id = message.actor m.created_at = message.created_at m.content = message.content m.text = None m.href = None return db_bz.insertIfNotExist(pg, 'm', m, "id_str='%s' and m_type='github'" % message.id_str)
def follow(user_id, god_id, make_sure=True): ''' create by bigzhu at 15/07/15 14:22:51 modify by bigzhu at 15/07/15 15:00:28 如果不用告警,就不要make_sure ''' id = db_bz.insertIfNotExist(pg, 'follow_who', {'user_id': user_id, 'god_id': god_id}, "user_id=%s and god_id=%s" % (user_id, god_id)) if id is None and make_sure: raise Exception('没有正确的Follow, 似乎已经Follow过了呢')
def post(self): self.set_header("Content-Type", "application/json") user_id = self.get_secure_cookie("user_id") for card_number in self.request.body.splitlines(): print "card_number:", card_number values = {"user_id": user_id, "card_number": card_number} id = db_bz.insertIfNotExist(pg, "available_card_numbers", values, where=" card_number='%s'" % card_number) print "insert ", values, "id:", id self.write(json.dumps({"error": "0"}, cls=public_bz.ExtEncoder))
def saveLast(last_time, last_message_id, user_id): ''' create by bigzhu at 15/08/16 16:22:39 保存最后一条的message ''' last_time = int(last_time) datetime_last_time = time_bz.timestampToDateTime(last_time, millisecond=True) id = db_bz.insertIfNotExist(pg, 'last', {'user_id': user_id, 'last_time': datetime_last_time, 'last_message_id': last_message_id}, "user_id=%s" % user_id) if id is None: count = pg.update('last', where='last_time< to_timestamp(%s/1000) and user_id=%s' % (last_time, user_id), last_message_id=last_message_id, last_time=datetime_last_time) return count return id
def follow(user_id, god_id, make_sure=True): ''' create by bigzhu at 15/07/15 14:22:51 modify by bigzhu at 15/07/15 15:00:28 如果不用告警,就不要make_sure ''' id = db_bz.insertIfNotExist(pg, 'follow_who', { 'user_id': user_id, 'god_id': god_id }, "user_id=%s and god_id=%s" % (user_id, god_id)) if id is None and make_sure: raise Exception('没有正确的Follow, 似乎已经Follow过了呢')
def post(self): self.set_header("Content-Type", "application/json") data = json.loads(self.request.body) id = db_bz.insertIfNotExist(pg, 'user_info', data, "user_name='%s'" % data['user_name']) if id is None: where = "user_name='%s'" % data['user_name'] self.pg.update("user_info", where=where, **data) id = self.pg.select('user_info', where=where)[0].id oper.follow(self.current_user, id, make_sure=False) self.write(json.dumps({'error': '0'}))
def saveMessage(message): ''' create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里 ''' message.id_str = message.pop('id') content = storage() content.type = message.type content.repo = message.pop('repo') content.payload = message.pop('payload') message.content = json.dumps(content) if message.get('org'): message.org = json.dumps(message.org) return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str)
def saveLast(last_message_id, user_id): ''' create by bigzhu at 15/08/16 16:22:39 保存最后一条的message ''' #last_time = int(last_time) #datetime_last_time = time_bz.timestampToDateTime(last_time, millisecond=True) id = db_bz.insertIfNotExist(pg, 'last', { 'user_id': user_id, 'last_message_id': last_message_id }, "user_id=%s" % user_id) if id is None: count = pg.update('last', where='last_message_id< %s and user_id=%s' % (last_message_id, user_id), last_message_id=last_message_id) return count return 1
def saveUser(id, url): ''' create by bigzhu at 15/07/15 21:27:19 保存github信息 create by bigzhu at 15/07/22 16:17:37 fix bug, not return id ''' if list(pg.select('github_user', where='id=%s' % id)): return id else: r = requests.get(url) user = storage(r.json()) del user.url del user.followers_url del user.following_url del user.gists_url del user.starred_url del user.subscriptions_url del user.organizations_url del user.repos_url del user.events_url del user.received_events_url del user.type return db_bz.insertIfNotExist(pg, 'github_user', user)
def saveUser(id, url): ''' create by bigzhu at 15/07/15 21:27:19 保存github信息 create by bigzhu at 15/07/22 16:17:37 fix bug, not return id ''' if list(pg.select('github_user', where='id=%s' % id)): return id else: r = requests.get(url) user = storage(r.json()) print user if hasattr(user, 'url'): del user.url del user.followers_url del user.following_url del user.gists_url del user.starred_url del user.subscriptions_url del user.organizations_url del user.repos_url del user.events_url del user.received_events_url del user.type return db_bz.insertIfNotExist(pg, 'github_user', user)
def insertIfNotExist(*args, **kwargs): return db_bz.insertIfNotExist(*args, **kwargs)
def getMedia(user_name=None, with_next_url=None, user=None): if user_name: user = getUser(user_name) if user is None: return # min_id 会查出大于等于这个id的 try: medias, next_ = api.user_recent_media(user_id=user.id, min_id=user.last_id) except instagram.bind.InstagramClientError: print public_bz.getExpInfoAll() public_db.delNoName('instagram', user_name) return if medias: last_id = medias[0].id pg.update('instagram_user', where="lower(username)=lower('%s')" % user_name, last_id=last_id) else: medias, next_ = api.user_recent_media(with_next_url=with_next_url) for media in medias: db_media = storage() if media.caption: caption = media.caption.__dict__ caption['user_id'] = caption['user'].id del caption['user'] else: caption = '' db_media.caption = json.dumps(caption, cls=public_bz.ExtEncoder) db_media.comment_count = media.comment_count if media.comments: media.comments = [d.__dict__ for d in media.comments] for comment in media.comments: comment['user'] = comment['user'].__dict__ db_media.comments = json.dumps(media.comments, cls=public_bz.ExtEncoder) db_media.created_time = media.created_time # 8小时的问题 db_media.created_time += timedelta(hours=8) db_media.filter = media.filter db_media.low_resolution = json.dumps(media.images['low_resolution'].__dict__) db_media.standard_resolution = json.dumps(media.images['standard_resolution'].__dict__) db_media.thumbnail = json.dumps(media.images['thumbnail'].__dict__) db_media.id_str = media.id db_media.like_count = media.like_count # likes里有User对象,暂时不存了 #db_media.likes = json.dumps(media.likes) db_media.link = media.link db_media.type = media.type db_media.user_id = user.id id = db_bz.insertIfNotExist(pg, 'instagram_media', db_media, "id_str='%s'" % db_media.id_str) print 'new=', media.id, user.username if id is not None and len(medias) <= 2: # 新增加消息,微信通知只通知2条以内 openids = public_db.getOpenidsByName('instagram', user.username) for data in openids: if caption != '': text = caption.get('text') else: text = '' wechat_oper.sendInstagram(data.openid, text, media.images['standard_resolution'].url, user.username, id) # 递归查出 if next_ != with_next_url: getMedia(with_next_url=next_, user=user)