Пример #1
0
def update_label(language=None, label=None):
    """
    If some feed in label change name, call this one
    """
    if not language or not label:
        return None

    feeds = Collection(db, FEED_REGISTRAR)
    # items = feeds.find({'language' = language, '%s.%s' % ('labels', label):
    #  {'$exists': True}})
    # This is incorrect!!!
    items = None
    if items:
        feeds_with_label = [item['feed_title'] for item in items]

        if feeds_with_label:
            label_name_in_memory = 'news::%s::%s' % (language, label)
            # remove all existing data regarding this label in memory
            if rclient.exsits(label_name_in_memory):
                label_total_in_memory = rclient.zcard(label_name_in_memory)
                # remove sequence in label_name_in_memory
                rclient.zremrangebyrank(label_name_in_memory, 0,
                                        label_total_in_memory)

                # reload entries of feeds to the sequence
                ids_all = rclient.keys('*')
                for id_in_memory in ids_all:
                    if 'news' not in id_in_memory and '::' not in id_in_memory:
                        item = eval(rclient.get(id_in_memory))
                        if item['feed'] in feeds_with_label:
                            rclient.zadd(label_name_in_memory, item['updated'],
                                         item['_id'])
                return 'OK'
    return None
Пример #2
0
def add_feed_to_label(language=None, feed=None, label=None, label_order=None):
    """
    Ad hoc add a feed to label. This mainly deals with history data
    """
    if not language or not feed or not label:
        return None

    # add label to the feed's labels
    feeds = Collection(db, FEED_REGISTRAR)
    item = feeds.find_one({'feed_title': feed})
    if item:
        if label not in item['labels']:
            item['labels'][label] = label_order
            feeds.update({'_id': item['_id']}.item)

            # add entries in memory of the feed
            label_name_in_memory = 'news::%s::%s' % (language, label)
            ids_all = rclient.keys('*')
            for id_in_memory in ids_all:
                if 'news' not in id_in_memory and '::' not in id_in_memory:
                    entry = eval(rclient.get(id_in_memory))
                    if entry['feed'] == feed:
                        rclient.zadd(
                            label_name_in_memory, entry['updated'],
                            entry['_id'])
            return 'OK'
        else:
            # label is already there
            return None
    else:
        return None
Пример #3
0
def remove_feed_from_label(language=None, feed=None, label=None):
    """
    Remove a feed from a label
    """
    if not language or not feed or not label:
        return None

    # remove member in label sequence in memory regarding the feed
    label_name_in_memory = 'news::%s::%s' % (language, label)
    if rclient.exists(label_name_in_memory):
        label_total_in_memory = rclient.zcard(label_name_in_memory)
        ids_with_label = rclient.zrevrange(
            label_name_in_memory, 0, label_total_in_memory)
        for id_with_label in ids_with_label:
            item = eval(rclient.get(id_with_label))
            if item['feed'] == feed:
                rclient.zrem(id_with_label)

    # remove label in feed
    feeds = Collection(db, FEED_REGISTRAR)
    item = feeds.find_one(
        {'feed_title': feed, '%s.%s' % ('labels', label): {'$exists': True}})
    if item:
        item['labels'].pop(label)
        feeds.update({'_id': item['_id']}, item)
        return 'OK'

    return None
Пример #4
0
def update_label(language=None, label=None):
    """
    If some feed in label change name, call this one
    """
    if not language or not label:
        return None

    feeds = Collection(db, FEED_REGISTRAR)
    # items = feeds.find({'language' = language, '%s.%s' % ('labels', label):
    #  {'$exists': True}})
    # This is incorrect!!!
    items = None
    if items:
        feeds_with_label = [item['feed_title'] for item in items]

        if feeds_with_label:
            label_name_in_memory = 'news::%s::%s' % (language, label)
            # remove all existing data regarding this label in memory
            if rclient.exsits(label_name_in_memory):
                label_total_in_memory = rclient.zcard(label_name_in_memory)
                # remove sequence in label_name_in_memory
                rclient.zremrangebyrank(label_name_in_memory, 0,
                                        label_total_in_memory)

                # reload entries of feeds to the sequence
                ids_all = rclient.keys('*')
                for id_in_memory in ids_all:
                    if 'news' not in id_in_memory and '::' not in id_in_memory:
                        item = eval(rclient.get(id_in_memory))
                        if item['feed'] in feeds_with_label:
                            rclient.zadd(label_name_in_memory, item['updated'],
                                         item['_id'])
                return 'OK'
    return None
Пример #5
0
def remove_feed_from_label(language=None, feed=None, label=None):
    """
    Remove a feed from a label
    """
    if not language or not feed or not label:
        return None

    # remove member in label sequence in memory regarding the feed
    label_name_in_memory = 'news::%s::%s' % (language, label)
    if rclient.exists(label_name_in_memory):
        label_total_in_memory = rclient.zcard(label_name_in_memory)
        ids_with_label = rclient.zrevrange(label_name_in_memory, 0,
                                           label_total_in_memory)
        for id_with_label in ids_with_label:
            item = eval(rclient.get(id_with_label))
            if item['feed'] == feed:
                rclient.zrem(id_with_label)

    # remove label in feed
    feeds = Collection(db, FEED_REGISTRAR)
    item = feeds.find_one({
        'feed_title': feed,
        '%s.%s' % ('labels', label): {
            '$exists': True
        }
    })
    if item:
        item['labels'].pop(label)
        feeds.update({'_id': item['_id']}, item)
        return 'OK'

    return None
Пример #6
0
def add_feed_to_label(language=None, feed=None, label=None, label_order=None):
    """
    Ad hoc add a feed to label. This mainly deals with history data
    """
    if not language or not feed or not label:
        return None

    # add label to the feed's labels
    feeds = Collection(db, FEED_REGISTRAR)
    item = feeds.find_one({'feed_title': feed})
    if item:
        if label not in item['labels']:
            item['labels'][label] = label_order
            feeds.update({'_id': item['_id']}.item)

            # add entries in memory of the feed
            label_name_in_memory = 'news::%s::%s' % (language, label)
            ids_all = rclient.keys('*')
            for id_in_memory in ids_all:
                if 'news' not in id_in_memory and '::' not in id_in_memory:
                    entry = eval(rclient.get(id_in_memory))
                    if entry['feed'] == feed:
                        rclient.zadd(label_name_in_memory, entry['updated'],
                                     entry['_id'])
            return 'OK'
        else:
            # label is already there
            return None
    else:
        return None
Пример #7
0
def clean():
    """
    remove expired items from queues in memory
    walk through all redis content
    """
    logger.info('... cleaning memory ...')
    try:
        news_lists = rclient.keys('news::*')
        for news_list in news_lists:
            # get the total number of a news list
            news_list_count = rclient.zcard(news_list)
            # get all the ids in a news list
            if news_list_count:
                news_ids = rclient.zrange(news_list, 0, news_list_count)
                for news_id in news_ids:
                    # make sure every item is touched
                    if not rclient.exists(news_id):
                        rclient.zrem(news_list, news_id)
                    else:
                        news_item_string = rclient.get(news_id)
                        if news_item_string:
                            news_item = eval(news_item_string)
                            news_updated = float(news_item['updated'])

                            # WTF, remove it
                            if cleaner.is_overdue(news_updated):
                                rclient.zrem(news_list, news_id)
                                rclient.delete(news_id)
                            else:  # check if this is zombie
                                if _is_zombie(news_item):
                                    rclient.zrem(news_list, news_id)
                                    rclient.delete(news_id)
                        else:
                            rclient.zrem(news_list, news_id)
                            rclient.delete(news_id)
            else:
                logger.error('Nothing in the list')
                continue
        return True
    except Exception as k:
        logger.error(str(k))
        return False
Пример #8
0
def modify_feed(language=None, feed_old=None, feed_new=None):
    """
    Call this once a feed's name is changed
    """
    if not language or not feed_old or not feed_new:
        return None

    # update old feed name in memory to new one
    feed_old_name_in_memory = 'news::%s::%s' % (language, feed_old)
    feed_new_name_in_memory = 'news::%s::%s' % (language, feed_new)

    ids_all = rclient.keys('*')
    for id_in_memory in ids_all:
        if 'news' not in id_in_memory and '::' not in id_in_memory:
            entry = eval(rclient.get(id_in_memory))
            if entry['feed'] == feed_old:
                entry['feed'] = feed_new
                rclient.set(name=id_in_memory,
                            value=entry,
                            ex=rclient.ttl(id_in_memory),
                            xx=True)
        elif id_in_memory == feed_old_name_in_memory:
            rclient.rename(feed_old_name_in_memory, feed_new_name_in_memory)
        else:
            continue

    # update old feed name in database to new one
    # change the "meta" data of feed_old in database
    feeds = Collection(db, FEED_REGISTRAR)
    item = feeds.find_one({'feed_title': feed_old})
    if item:
        item['feed_title'] = feed_new
        feeds.update({'_id': item['_id']}, item)
    # change all the related news in database
    col = Collection(db, language)
    items = col.find({'feed': feed_old})
    if items:
        for item in items:
            item['feed'] = feed_new
            col.update({'_id': item['_id']}, item)

    return 'OK'
Пример #9
0
def modify_feed(language=None, feed_old=None, feed_new=None):
    """
    Call this once a feed's name is changed
    """
    if not language or not feed_old or not feed_new:
        return None

    # update old feed name in memory to new one
    feed_old_name_in_memory = 'news::%s::%s' % (language, feed_old)
    feed_new_name_in_memory = 'news::%s::%s' % (language, feed_new)

    ids_all = rclient.keys('*')
    for id_in_memory in ids_all:
        if 'news' not in id_in_memory and '::' not in id_in_memory:
            entry = eval(rclient.get(id_in_memory))
            if entry['feed'] == feed_old:
                entry['feed'] = feed_new
                rclient.set(
                    name=id_in_memory, value=entry,
                    ex=rclient.ttl(id_in_memory), xx=True)
        elif id_in_memory == feed_old_name_in_memory:
            rclient.rename(feed_old_name_in_memory, feed_new_name_in_memory)
        else:
            continue

    # update old feed name in database to new one
    # change the "meta" data of feed_old in database
    feeds = Collection(db, FEED_REGISTRAR)
    item = feeds.find_one({'feed_title': feed_old})
    if item:
        item['feed_title'] = feed_new
        feeds.update({'_id': item['_id']}, item)
    # change all the related news in database
    col = Collection(db, language)
    items = col.find({'feed': feed_old})
    if items:
        for item in items:
            item['feed'] = feed_new
            col.update({'_id': item['_id']}, item)

    return 'OK'
Пример #10
0
def modify_field(language):
    """
    change a key/field name in database and memory
    """
    # memory
    id_total = rclient.zcard("news::%s" % language)
    ids = rclient.zrange("news::%s" % language, 0, id_total)

    for id in ids:
        entry_string = rclient.get(id)
        entry_string_new = entry_string.replace(
            'hot_news_image', 'hotnews_image')
        rclient.setex(id, rclient.ttl(id), entry_string_new)

    # database
    col = Collection(db, language)
    col.update(
        {}, {"rename": {"hot_news_image": "hotnews_image"}}, False, True)
    col.update(
        {}, {"rename": {"hot_news_image_local": "hotnews_image_local"}}, False,
          True)
Пример #11
0
def modify_field(language):
    """
    change a key/field name in database and memory
    """
    # memory
    id_total = rclient.zcard("news::%s" % language)
    ids = rclient.zrange("news::%s" % language, 0, id_total)

    for id in ids:
        entry_string = rclient.get(id)
        entry_string_new = entry_string.replace('hot_news_image',
                                                'hotnews_image')
        rclient.setex(id, rclient.ttl(id), entry_string_new)

    # database
    col = Collection(db, language)
    col.update({}, {"rename": {
        "hot_news_image": "hotnews_image"
    }}, False, True)
    col.update({}, {"rename": {
        "hot_news_image_local": "hotnews_image_local"
    }}, False, True)