def get(self):
        queue = getQueue()
        stat = getStat()
        now = datetime.now()
        while len(queue) > 0 and now - queue[0][1] > timedelta(hours=1):
            if queue[0][0] in stat:
                stat[queue[0][0]] = stat[queue[0][0]] - 1
                if stat[queue[0][0]] == 0:
                    del (stat[queue[0][0]])
            else:
                logging.error(
                    "Fatal error with trending stream data. Data in queue but does not in stat."
                )
            queue.popleft()
        trending = [(x.key.id(), stat[x.key.id()] if x.key.id() in stat else 0)
                    for x in Stream.query(
                        ancestor=getStreamKey()).order(-Stream.viewCount)]
        trending.sort(key=lambda x: -x[1])
        updateQueue(queue)
        updateStat(stat)
        updateTrendingStreams(trending)
        n = 0
        trendingStreams = []
        for stat in trending:
            stream = Stream.get_by_id(stat[0], getStreamKey())
            if stream:
                trendingStreams.append((stream, stat[1]))
                n = n + 1
                if n == Trending_Stream_Handler.digest_num:
                    break
        for frequency in Trending_Stream_Handler.email_digest_counter.keys():
            Trending_Stream_Handler.email_digest_counter[
                frequency] = Trending_Stream_Handler.email_digest_counter[
                    frequency] + 1
            if Trending_Stream_Handler.email_digest_counter[
                    frequency] == Trending_Stream_Handler.email_digest_bound[
                        frequency]:
                Trending_Stream_Handler.email_digest_counter[frequency] = 0
                for userSetting in UserEmailSetting.query(
                        UserEmailSetting.email_update_rate == frequency):
                    emails.sendTrendingStreamEmails(userSetting.user,
                                                    userSetting.name,
                                                    trendingStreams)

        self.response.write(str(trending))
def getTrendingStreams():
    trending = memcache.get('TrendingStat')
    if trending:
        return trending
    trending = [
        (x.key.id(), 0)
        for x in Stream.query(ancestor=getStreamKey()).order(Stream.viewCount)
    ]
    updateTrendingStreams(trending)
    return trending
def getTopStreams(max_num):
    trending = getTrendingStreams()
    n = 0
    trendingStreams = []
    for stat in trending:
        stream = Stream.get_by_id(stat[0], getStreamKey())
        if stream:
            trendingStreams.append((stream, stat[1]))
            n = n + 1
            if n == max_num:
                break
    return trendingStreams
Beispiel #4
0
def getTrie():
    trie = memcache.get('search')
    if trie:
        return trie
    trie = Trie()
    for stream in Stream.query(ancestor=getStreamKey()):
        sid = stream.key.id()
        for str in formatSearchContent(stream.name):
            trie.add(str, sid)
        for str in formatSearchContent(stream.tag):
            trie.add(str, sid)
        for str in formatSearchContent("".join(
                re.findall(r'(.+)@', stream.user))):
            trie.add(str, sid)
        updateTrie(trie)
    return trie
Beispiel #5
0
def search_streams(query_words):
    query_words = formatSearchContent(query_words)
    sids = []
    relevence = {}
    trie = getTrie()
    for word in query_words:
        sids_of_word = set()
        for string in trie.searchSubstring(word):
            sids_of_string = trie.get(string)
            for sid in sids_of_string:
                if not (sid in relevence):
                    relevence[sid] = 0.0
                relevence[sid] = relevence[sid] + float(
                    len(word)) / len(string)
            sids_of_word = sids_of_word.union(sids_of_string)
        sids.append(sids_of_word)
    ret = [
        Stream.get_by_id(sid, getStreamKey())
        for sid in reduce(lambda x, y: x.intersection(y), sids)
    ]
    return sorted(ret, key=lambda x: -relevence[x.key.id()])