def broadcastTopic(topicTitle): try: logger.debug(u'Start broadcast topic:%s' % topicTitle) topic = djangodb.Topic.objects.get(title=topicTitle) if topic.alive(): topic_news = topic.news_set.all()[0] else: logger.warn(u'topic:%s is already dead, add unsubscribe task!' % topicTitle) djangodb.add_task(topic=topic, type=u'unsubscribe') return False except: logger.exception(u'topic or news may not exist in database') return False alreadyFetchedWeibos = topic.relevant_weibo.all() fetchedWeibos = weibo.getRelevantWeibo(topicTitle=topicTitle, count=50)['statuses'] logger.info(u'feteched %d weibo about topic:%s' % (len(fetchedWeibos), topicTitle)) for awb in fetchedWeibos: aweibo = djangodb.get_or_create_weibo(awb) if aweibo.comments_count < 200 and aweibo in alreadyFetchedWeibos: # 这条微薄已经推广过了,由于时间关系,后续的微薄理论上也应该推广过了 continue break else: topic.relevant_weibo.add(aweibo) auser = djangodb.get_or_create_account_from_weibo(awb['user']) if aweibo.comments_count < 200 and auser in topic.watcher.all(): # 该用户已经使用服务,不需要推广 continue if aweibo.comments_count < 200 and len(auser.remind_history.strip()) > 0: # 已经推广过至少一次或正在使用服务 continue topic.relevant_user.add(auser) # 评论推广 _shorturl = weibo.getShortUrl(Conf.get_timeline_url(topic.id)) reqInterval(10) msg = u'#%s# %s 『%s』 ^ ^' % (topicTitle, topic_news.title, _shorturl) try: weibo.postComment(weibo_id=aweibo.weibo_id, content=msg) except APIError, err: logger.warn(u'comment failed:\t%s' % (err.error)) if err.error == u'target weibo does not exist!': topic.relevant_user.remove(auser) topic.relevant_weibo.remove(aweibo) aweibo.delete() logger.info(u'remove relevant user:%s and delete weibo:%s' % (auser, aweibo)) except:
def fetchRssUpdates(): ''' 检查更新rss订阅源,并存储更新的新闻,之后标记rss为已读 ''' logger.info(u'\nStart fetch rss update\n') unreadFeedsDict = reader.getUnreadFeeds() logger.info(u'get %d unread feeds' % len(unreadFeedsDict)) reqInterval() for feed in unreadFeedsDict.keys(): if(feed.startswith(u'feed')): excludeRead = True continuation = None topic = None over = False while not over: try: feedContent = reader.fetchFeedItems(feed, excludeRead, continuation) # title的形式:"镜头里的萝莉 - Google 新闻" 要截断后面的 feedTopic = feedContent[u'title'][0:-12] itemSet = feedContent[u'items'] except: logger.exception(u'failed to fetch feed items, sleep 123s and retry again') reqInterval(123) continue else: logger.info('Fetch %d feed items of topic %s' % (len(itemSet), feedTopic)) reqInterval(31) try: continuation = feedContent[u'continuation'] if not continuation: over = True except: logger.info(u'fail to extract continuation, may well be normal(over)') over = True else: logger.info('Extract continuation:%s' % continuation) try: topic = djangodb.Topic.objects.get(title=feedTopic) if not topic.alive(): logger.warn(u'topic %s is already dead, unsubscribe!' % topic.title) djangodb.add_task(topic=topic, type=u'unsubscribe') break except: logger.exception(u'topic:%s not found in database, manually unsubscribe!' % feedTopic) break for item in itemSet: title = item[u'title'] pubDate = datetime.fromtimestamp(float(item[u'published'])) try: summary = item[u'summary'] except: logger.exception(u'fail to extract summary from: %s' % item) summary = u'Fail to extract summary==!' link = item[u'alternate'][0][u'href'] if '&url=http' in link: link = link[link.find(u'&url=http') + 5:] nnews, created = djangodb.News.objects.get_or_create(title=title) if created: nnews.link = link nnews.pubDate = pubDate nnews.summary = summary nnews.topic.add(topic) nnews.save() logger.info("add news:%s to topic:%s" % (nnews.title, topic.title)) if topic: # 标记该feed为全部已读 try: sucInMarkAsRead = reader.markFeedAsRead(feed) except: sucInMarkAsRead = False logger.exception(u'Fail to mark feedtopic:%s as read!!!' % feedTopic) finally: if sucInMarkAsRead: logger.info(u'Succeed to mark feedtopic:%s as read!!!' % feedTopic) else: logger.error(u'Fail to mark feedtopic:%s as read!!!' % feedTopic) reqInterval(31) # 更新话题的news timeline logger.info(u'Update news timeline for topic:%s' % feedTopic) create_or_update_news_timeline(feedTopic) # 添加提醒任务 logger.debug(u'Add remind task for topic:%s' % feedTopic) djangodb.add_task(topic=topic, type=u'remind') logger.debug(u'\nFetch rss update over\n')
ch = logging.StreamHandler() ch.setLevel(logging.WARN) # create logger output formater formatter = logging.Formatter(u'%(asctime)s %(name)s %(levelname)s %(message)s') fh.setFormatter(formatter) ch.setFormatter(formatter) logger.addHandler(fh) logger.addHandler(ch) # setup google reader from libgreader import readerAPI [access_token, refresh_token, access_expires] = djangodb.get_google_auth_info(u_id=1) reader = readerAPI.readerAPI(u_id=1, access_token=access_token, \ refresh_token=refresh_token, expires_access=access_expires) logger.info(u'Google Reader 登录信息:%s' % reader.getUserInfo()[u'userName']) reqInterval(31) def fetchRssUpdates(): ''' 检查更新rss订阅源,并存储更新的新闻,之后标记rss为已读 ''' logger.info(u'\nStart fetch rss update\n') unreadFeedsDict = reader.getUnreadFeeds() logger.info(u'get %d unread feeds' % len(unreadFeedsDict)) reqInterval() for feed in unreadFeedsDict.keys(): if(feed.startswith(u'feed')): excludeRead = True continuation = None topic = None
def remindUserTopicUpdates(topicTitle): try: logger.debug(u'Start remind user for topic:%s' % topicTitle) topic = djangodb.Topic.objects.get(title=topicTitle) if topic.alive(): topic_news = topic.news_set.all()[0] else: logger.warn(u'topic:%s is already dead, add unsubscribe task!' % topicTitle) djangodb.add_task(topic=topic, type=u'unsubscribe') return False except: logger.exception(u'topic or news may not exist in database') return False topicWatchers = topic.watcher.all() topicWatcherWeibo = topic.watcher_weibo.all() ''' 得到订阅该话题的所有用户,分两类,已经授权的和没有授权的 ''' watcherWithAuth = set() watcherWithoutAuth = set() for watcher in topicWatchers: if watcher.has_oauth(): watcherWithAuth.add(watcher) else: watcherWithoutAuth.add(watcher) ''' 筛选出其中需要提醒的用户,分为四类: watcherWithoutStatusAndAuth watcherWithStatusAndAuth watcherWithStatusWithoutAuth watcherWithoutStatusWithAuth ''' watcherWithStatusAndAuth = set() watcherWithStatusWithoutAuth = set() for watcherWeibo in topicWatcherWeibo: watcher = watcherWeibo.user if not watcher.to_remind(): # 去掉不需要提醒的用户 continue watcher.original_weibo = watcherWeibo # 人工添加的字段 if watcher in watcherWithAuth: watcherWithStatusAndAuth.add(watcher) elif watcher in watcherWithoutAuth: watcherWithStatusWithoutAuth.add(watcher) # 去掉不需要提醒的用户 watcherWithoutStatusAndAuth = set([watcher for watcher in (watcherWithoutAuth - watcherWithStatusWithoutAuth) if watcher.to_remind()]) watcherWithoutStatusWithAuth = set([watcher for watcher in (watcherWithAuth - watcherWithStatusAndAuth) if watcher.to_remind()]) _shorturl = weibo.getShortUrl(Conf.get_timeline_url(topic.id)) _msg = u'#%s#有新进展:%s 『%s』' % (topicTitle, topic_news.title, _shorturl) reqInterval(61) logger.debug(u'topicWatcherWeibo:%s' % topicWatcherWeibo) logger.debug(u'topicWatchers:%s' % topicWatchers) logger.debug(u'watcherWithAuth:%s' % watcherWithAuth) logger.debug(u'watcherWithoutAuth:%s' % watcherWithoutAuth) logger.debug(u'posgMsg:%s' % _msg) _user_reminded = [] ''' 先更新有授权同时也发过微博的用户的状态 ''' for watcher in watcherWithStatusAndAuth: ''' 自己转发或评论自己的原始微博提醒自己,如果允许提醒他人的话,顺带提醒~ ''' [access_token, expires_in] = djangodb.get_weibo_auth_info(watcher.weiboId) _weibo = weiboAPI.weiboAPI(access_token=access_token, \ expires_in=expires_in, \ u_id=watcher.weiboId) watcher_btw = None if watcher.allow_remind_others and watcherWithStatusWithoutAuth: watcher_btw = watcherWithStatusWithoutAuth.pop() postMsg = _msg + u' @' + watcher_btw.weiboName + u' 顺便提醒你一下~' else: postMsg = _msg + u' @' + watcher.weiboName logger.info(u'remind user:%s \ntopic:%s \nupdate with msg:%s\noriginalWeibo:%s' % \ (watcher.weiboName, topicTitle, postMsg, watcher.original_weibo)) res = {} try: if watcher.repost_remind: res[u'type'] = u'repost status' res[u'status'] = _weibo.repostStatus(weibo_id=watcher.original_weibo.weibo_id, content=postMsg) else: res[u'type'] = u'comment status' res[u'status'] = _weibo.postComment(weibo_id=watcher.original_weibo.weibo_id, content=postMsg) except APIError, err: logger.warn(u'%s failed:\t%s' % (res[u'type'], err.error)) if err.error == u'target weibo does not exist!': topic.watcher_weibo.remove(watcher.original_weibo) topic.watcher.remove(watcher) logger.info(u'remove watcher:%s and delete watcherWeibo:%s' % (watcher, watcher.original_weibo)) watcher.original_weibo.delete() else: logger.exception(u'') else: logger.info(u'%s Succeed!' % res[u'type']) if watcher_btw: watcher_btw.add_remind() watcher.add_remind() logger.info(u'added remind history for user: [%s, %s]' % (watcher.weiboName, watcher_btw)) finally:
# create console handler with warn log level ch = logging.StreamHandler() ch.setLevel(logging.WARN) # create logger output formater formatter = logging.Formatter(u'%(asctime)s %(name)s %(levelname)s %(message)s') fh.setFormatter(formatter) ch.setFormatter(formatter) logger.addHandler(fh) logger.addHandler(ch) # setup weibo [access_token, expires_in] = djangodb.get_weibo_auth_info(3041970403) if time.time() > float(expires_in): raise Exception(u'授权过期了,with expires_in:%s' % expires_in) weibo = weiboAPI.weiboAPI(access_token=access_token, expires_in=expires_in, u_id=3041970403) reqInterval(31) logger.info(u'Sina Weibo 登录信息:\t' + weibo.getUserInfo()[u'name']) # setup google reader from libgreader import readerAPI [access_token, refresh_token, access_expires] = djangodb.get_google_auth_info(u_id=1) reader = readerAPI.readerAPI(u_id=1, access_token=access_token, \ refresh_token=refresh_token, expires_access=access_expires) reqInterval(31) logger.info(u'Google Reader 登录信息:\t' + reader.getUserInfo()[u'userName']) def remindUserTopicUpdates(topicTitle): try: logger.debug(u'Start remind user for topic:%s' % topicTitle) topic = djangodb.Topic.objects.get(title=topicTitle)
# create console handler with warn log level ch = logging.StreamHandler() ch.setLevel(logging.WARN) # create logger output formater formatter = logging.Formatter(u"%(asctime)s %(name)s %(levelname)s %(message)s") fh.setFormatter(formatter) ch.setFormatter(formatter) logger.addHandler(fh) logger.addHandler(ch) # setup weibo [access_token, expires_in] = djangodb.get_weibo_auth_info(3041970403) if time.time() > float(expires_in): raise Exception(u"授权过期了,with expires_in:%s" % str(expires_in)) weibo = weiboAPI.weiboAPI(access_token=access_token, expires_in=expires_in, u_id=3041970403) reqInterval(31) logger.info(u"Sina Weibo 登录信息:%s" % weibo.getUserInfo()[u"name"]) def fetchUserMention(): """ 获取微博上用户的订阅话题,之后更新订阅 用户订阅话题的方式可以发布包含:@狗狗追踪 *订阅或取消订阅* #话题内容#的微博;或者转发上述微博 TODO: 1. mentions只获取了前50条 """ # 得到用户mentions lastMentionId = djangodb.get_last_mention_id() logger.info(u"Get lastMentionId:%s" % str(lastMentionId)) try: