Example #1
0
def broadcastTopic(topicTitle):
    try:
        logger.debug(u'Start broadcast topic:%s' % topicTitle)
        topic = djangodb.Topic.objects.get(title=topicTitle)
        if topic.alive():
            topic_news = topic.news_set.all()[0]
        else:
            logger.warn(u'topic:%s is already dead, add unsubscribe task!' % topicTitle)
            djangodb.add_task(topic=topic, type=u'unsubscribe')
            return False
    except:
        logger.exception(u'topic or news may not exist in database')
        return False

    alreadyFetchedWeibos = topic.relevant_weibo.all()

    fetchedWeibos = weibo.getRelevantWeibo(topicTitle=topicTitle, count=50)['statuses']
    logger.info(u'feteched %d weibo about topic:%s' % (len(fetchedWeibos), topicTitle))
    for awb in fetchedWeibos:
        aweibo = djangodb.get_or_create_weibo(awb)
        if aweibo.comments_count < 200 and aweibo in alreadyFetchedWeibos:
            # 这条微薄已经推广过了,由于时间关系,后续的微薄理论上也应该推广过了
            continue
            break
        else:
            topic.relevant_weibo.add(aweibo)

        auser = djangodb.get_or_create_account_from_weibo(awb['user'])
        if aweibo.comments_count < 200 and auser in topic.watcher.all():
            # 该用户已经使用服务,不需要推广
            continue
        if aweibo.comments_count < 200 and len(auser.remind_history.strip()) > 0:
            # 已经推广过至少一次或正在使用服务
            continue

        topic.relevant_user.add(auser)
        # 评论推广
        _shorturl = weibo.getShortUrl(Conf.get_timeline_url(topic.id))
        reqInterval(10)
        msg = u'#%s# %s 『%s』 ^ ^' % (topicTitle, topic_news.title, _shorturl)

        try:
            weibo.postComment(weibo_id=aweibo.weibo_id, content=msg)
        except APIError, err:
            logger.warn(u'comment failed:\t%s' % (err.error))
            if err.error == u'target weibo does not exist!':
                topic.relevant_user.remove(auser)
                topic.relevant_weibo.remove(aweibo)
                aweibo.delete()
                logger.info(u'remove relevant user:%s and delete weibo:%s' % (auser, aweibo))
        except:
Example #2
0
def fetchRssUpdates():
    '''
    检查更新rss订阅源,并存储更新的新闻,之后标记rss为已读
    '''
    logger.info(u'\nStart fetch rss update\n')
    unreadFeedsDict = reader.getUnreadFeeds()
    logger.info(u'get %d unread feeds' % len(unreadFeedsDict))
    reqInterval()
    for feed in unreadFeedsDict.keys():
        if(feed.startswith(u'feed')):
            excludeRead = True
            continuation = None
            topic = None
            over = False
            while not over:
                try:
                    feedContent = reader.fetchFeedItems(feed, excludeRead, continuation)
                    # title的形式:"镜头里的萝莉 - Google 新闻"  要截断后面的
                    feedTopic = feedContent[u'title'][0:-12]
                    itemSet = feedContent[u'items']
                except:
                    logger.exception(u'failed to fetch feed items, sleep 123s and retry again')
                    reqInterval(123)
                    continue
                else:
                    logger.info('Fetch %d feed items of topic %s' % (len(itemSet), feedTopic))
                    reqInterval(31)

                try:
                    continuation = feedContent[u'continuation']
                    if not continuation:
                        over = True
                except:
                    logger.info(u'fail to extract continuation, may well be normal(over)')
                    over = True
                else:
                    logger.info('Extract continuation:%s' % continuation)


                try:
                    topic = djangodb.Topic.objects.get(title=feedTopic)
                    if not topic.alive():
                        logger.warn(u'topic %s is already dead, unsubscribe!' % topic.title)
                        djangodb.add_task(topic=topic, type=u'unsubscribe')
                        break
                except:
                    logger.exception(u'topic:%s not found in database, manually unsubscribe!' % feedTopic)
                    break

                for item in itemSet:
                    title = item[u'title']
                    pubDate = datetime.fromtimestamp(float(item[u'published']))
                    try:
                        summary = item[u'summary']
                    except:
                        logger.exception(u'fail to extract summary from: %s' % item)
                        summary = u'Fail to extract summary==!'
                    link = item[u'alternate'][0][u'href']
                    if '&url=http' in link:
                        link = link[link.find(u'&url=http') + 5:]
                    nnews, created = djangodb.News.objects.get_or_create(title=title)
                    if created:
                        nnews.link = link
                        nnews.pubDate = pubDate
                        nnews.summary = summary
                    nnews.topic.add(topic)
                    nnews.save()
                    logger.info("add news:%s to topic:%s" % (nnews.title, topic.title))

            if topic:
                # 标记该feed为全部已读
                try:
                    sucInMarkAsRead = reader.markFeedAsRead(feed)
                except:
                    sucInMarkAsRead = False
                    logger.exception(u'Fail to mark feedtopic:%s as read!!!' % feedTopic)
                finally:
                    if sucInMarkAsRead:
                        logger.info(u'Succeed to mark feedtopic:%s as read!!!' % feedTopic)
                    else:
                        logger.error(u'Fail to mark feedtopic:%s as read!!!' % feedTopic)
                    reqInterval(31)

                #  更新话题的news timeline
                logger.info(u'Update news timeline for topic:%s' % feedTopic)
                create_or_update_news_timeline(feedTopic)

                # 添加提醒任务
                logger.debug(u'Add remind task for topic:%s' % feedTopic)
                djangodb.add_task(topic=topic, type=u'remind')

    logger.debug(u'\nFetch rss update over\n')
Example #3
0
ch = logging.StreamHandler()
ch.setLevel(logging.WARN)
# create logger output formater
formatter = logging.Formatter(u'%(asctime)s %(name)s %(levelname)s %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)

# setup google reader
from libgreader import readerAPI
[access_token, refresh_token, access_expires] = djangodb.get_google_auth_info(u_id=1)
reader = readerAPI.readerAPI(u_id=1, access_token=access_token, \
                   refresh_token=refresh_token, expires_access=access_expires)
logger.info(u'Google Reader 登录信息:%s' % reader.getUserInfo()[u'userName'])
reqInterval(31)


def fetchRssUpdates():
    '''
    检查更新rss订阅源,并存储更新的新闻,之后标记rss为已读
    '''
    logger.info(u'\nStart fetch rss update\n')
    unreadFeedsDict = reader.getUnreadFeeds()
    logger.info(u'get %d unread feeds' % len(unreadFeedsDict))
    reqInterval()
    for feed in unreadFeedsDict.keys():
        if(feed.startswith(u'feed')):
            excludeRead = True
            continuation = None
            topic = None
Example #4
0
def remindUserTopicUpdates(topicTitle):
    try:
        logger.debug(u'Start remind user for topic:%s' % topicTitle)
        topic = djangodb.Topic.objects.get(title=topicTitle)
        if topic.alive():
            topic_news = topic.news_set.all()[0]
        else:
            logger.warn(u'topic:%s is already dead, add unsubscribe task!' % topicTitle)
            djangodb.add_task(topic=topic, type=u'unsubscribe')
            return False
    except:
        logger.exception(u'topic or news may not exist in database')
        return False

    topicWatchers = topic.watcher.all()
    topicWatcherWeibo = topic.watcher_weibo.all()


    '''
    得到订阅该话题的所有用户,分两类,已经授权的和没有授权的
    '''
    watcherWithAuth = set()
    watcherWithoutAuth = set()
    for watcher in topicWatchers:
        if watcher.has_oauth():
            watcherWithAuth.add(watcher)
        else:
            watcherWithoutAuth.add(watcher)

    '''
    筛选出其中需要提醒的用户,分为四类:
    
    watcherWithoutStatusAndAuth
    watcherWithStatusAndAuth
    watcherWithStatusWithoutAuth
    watcherWithoutStatusWithAuth
    '''
    watcherWithStatusAndAuth = set()
    watcherWithStatusWithoutAuth = set()
    for watcherWeibo in topicWatcherWeibo:
        watcher = watcherWeibo.user
        if not watcher.to_remind():
            # 去掉不需要提醒的用户
            continue
        watcher.original_weibo = watcherWeibo  # 人工添加的字段
        if watcher in watcherWithAuth:
            watcherWithStatusAndAuth.add(watcher)
        elif watcher in watcherWithoutAuth:
            watcherWithStatusWithoutAuth.add(watcher)

    # 去掉不需要提醒的用户
    watcherWithoutStatusAndAuth = set([watcher for watcher in (watcherWithoutAuth - watcherWithStatusWithoutAuth) if watcher.to_remind()])
    watcherWithoutStatusWithAuth = set([watcher for watcher in (watcherWithAuth - watcherWithStatusAndAuth) if watcher.to_remind()])

    _shorturl = weibo.getShortUrl(Conf.get_timeline_url(topic.id))
    _msg = u'#%s#有新进展:%s 『%s』' % (topicTitle, topic_news.title, _shorturl)
    reqInterval(61)

    logger.debug(u'topicWatcherWeibo:%s' % topicWatcherWeibo)
    logger.debug(u'topicWatchers:%s' % topicWatchers)
    logger.debug(u'watcherWithAuth:%s' % watcherWithAuth)
    logger.debug(u'watcherWithoutAuth:%s' % watcherWithoutAuth)
    logger.debug(u'posgMsg:%s' % _msg)

    _user_reminded = []

    '''
    先更新有授权同时也发过微博的用户的状态
    '''
    for watcher in watcherWithStatusAndAuth:
        '''
        自己转发或评论自己的原始微博提醒自己,如果允许提醒他人的话,顺带提醒~
        '''
        [access_token, expires_in] = djangodb.get_weibo_auth_info(watcher.weiboId)
        _weibo = weiboAPI.weiboAPI(access_token=access_token, \
                                   expires_in=expires_in, \
                                   u_id=watcher.weiboId)
        watcher_btw = None
        if watcher.allow_remind_others and watcherWithStatusWithoutAuth:
            watcher_btw = watcherWithStatusWithoutAuth.pop()
            postMsg = _msg + u' @' + watcher_btw.weiboName + u' 顺便提醒你一下~'
        else:
            postMsg = _msg + u' @' + watcher.weiboName

        logger.info(u'remind user:%s \ntopic:%s \nupdate with msg:%s\noriginalWeibo:%s' % \
                     (watcher.weiboName, topicTitle, postMsg, watcher.original_weibo))
        res = {}
        try:
            if watcher.repost_remind:
                res[u'type'] = u'repost status'
                res[u'status'] = _weibo.repostStatus(weibo_id=watcher.original_weibo.weibo_id, content=postMsg)
            else:
                res[u'type'] = u'comment status'
                res[u'status'] = _weibo.postComment(weibo_id=watcher.original_weibo.weibo_id, content=postMsg)
        except APIError, err:
            logger.warn(u'%s failed:\t%s' % (res[u'type'], err.error))
            if err.error == u'target weibo does not exist!':
                topic.watcher_weibo.remove(watcher.original_weibo)
                topic.watcher.remove(watcher)
                logger.info(u'remove watcher:%s and delete watcherWeibo:%s' % (watcher, watcher.original_weibo))
                watcher.original_weibo.delete()
            else:
                logger.exception(u'')
        else:
            logger.info(u'%s Succeed!' % res[u'type'])
            if watcher_btw:
                watcher_btw.add_remind()
            watcher.add_remind()
            logger.info(u'added remind history for user: [%s, %s]' % (watcher.weiboName, watcher_btw))
        finally:
Example #5
0
# create console handler with warn log level
ch = logging.StreamHandler()
ch.setLevel(logging.WARN)
# create logger output formater
formatter = logging.Formatter(u'%(asctime)s %(name)s %(levelname)s %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)

# setup weibo
[access_token, expires_in] = djangodb.get_weibo_auth_info(3041970403)
if time.time() > float(expires_in):
    raise Exception(u'授权过期了,with expires_in:%s' % expires_in)
weibo = weiboAPI.weiboAPI(access_token=access_token, expires_in=expires_in, u_id=3041970403)
reqInterval(31)
logger.info(u'Sina Weibo 登录信息:\t' + weibo.getUserInfo()[u'name'])

# setup google reader
from libgreader import readerAPI
[access_token, refresh_token, access_expires] = djangodb.get_google_auth_info(u_id=1)
reader = readerAPI.readerAPI(u_id=1, access_token=access_token, \
                  refresh_token=refresh_token, expires_access=access_expires)
reqInterval(31)
logger.info(u'Google Reader 登录信息:\t' + reader.getUserInfo()[u'userName'])


def remindUserTopicUpdates(topicTitle):
    try:
        logger.debug(u'Start remind user for topic:%s' % topicTitle)
        topic = djangodb.Topic.objects.get(title=topicTitle)
Example #6
0
# create console handler with warn log level
ch = logging.StreamHandler()
ch.setLevel(logging.WARN)
# create logger output formater
formatter = logging.Formatter(u"%(asctime)s %(name)s %(levelname)s %(message)s")
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)

# setup weibo
[access_token, expires_in] = djangodb.get_weibo_auth_info(3041970403)
if time.time() > float(expires_in):
    raise Exception(u"授权过期了,with expires_in:%s" % str(expires_in))
weibo = weiboAPI.weiboAPI(access_token=access_token, expires_in=expires_in, u_id=3041970403)
reqInterval(31)
logger.info(u"Sina Weibo 登录信息:%s" % weibo.getUserInfo()[u"name"])


def fetchUserMention():
    """
    获取微博上用户的订阅话题,之后更新订阅
    用户订阅话题的方式可以发布包含:@狗狗追踪 *订阅或取消订阅* #话题内容#的微博;或者转发上述微博

    TODO:
    1. mentions只获取了前50条
    """
    # 得到用户mentions
    lastMentionId = djangodb.get_last_mention_id()
    logger.info(u"Get lastMentionId:%s" % str(lastMentionId))
    try: