def update_label(language=None, label=None):
"""
If some feed in label change name, call this one
"""
if not language or not label:
return None
feeds = Collection(db, FEED_REGISTRAR)
# items = feeds.find({'language' = language, '%s.%s' % ('labels', label):
# {'$exists': True}})
# This is incorrect!!!
items = None
if items:
feeds_with_label = [item['feed_title'] for item in items]
if feeds_with_label:
label_name_in_memory = 'news::%s::%s' % (language, label)
# remove all existing data regarding this label in memory
if rclient.exsits(label_name_in_memory):
label_total_in_memory = rclient.zcard(label_name_in_memory)
# remove sequence in label_name_in_memory
rclient.zremrangebyrank(label_name_in_memory, 0,
label_total_in_memory)
# reload entries of feeds to the sequence
ids_all = rclient.keys('*')
for id_in_memory in ids_all:
if 'news' not in id_in_memory and '::' not in id_in_memory:
item = eval(rclient.get(id_in_memory))
if item['feed'] in feeds_with_label:
rclient.zadd(label_name_in_memory, item['updated'],
item['_id'])
return 'OK'
return None
def add_feed_to_label(language=None, feed=None, label=None, label_order=None):
"""
Ad hoc add a feed to label. This mainly deals with history data
"""
if not language or not feed or not label:
return None
# add label to the feed's labels
feeds = Collection(db, FEED_REGISTRAR)
item = feeds.find_one({'feed_title': feed})
if item:
if label not in item['labels']:
item['labels'][label] = label_order
feeds.update({'_id': item['_id']}.item)
# add entries in memory of the feed
label_name_in_memory = 'news::%s::%s' % (language, label)
ids_all = rclient.keys('*')
for id_in_memory in ids_all:
if 'news' not in id_in_memory and '::' not in id_in_memory:
entry = eval(rclient.get(id_in_memory))
if entry['feed'] == feed:
rclient.zadd(
label_name_in_memory, entry['updated'],
entry['_id'])
return 'OK'
else:
# label is already there
return None
else:
return None
def remove_feed_from_label(language=None, feed=None, label=None):
"""
Remove a feed from a label
"""
if not language or not feed or not label:
return None
# remove member in label sequence in memory regarding the feed
label_name_in_memory = 'news::%s::%s' % (language, label)
if rclient.exists(label_name_in_memory):
label_total_in_memory = rclient.zcard(label_name_in_memory)
ids_with_label = rclient.zrevrange(
label_name_in_memory, 0, label_total_in_memory)
for id_with_label in ids_with_label:
item = eval(rclient.get(id_with_label))
if item['feed'] == feed:
rclient.zrem(id_with_label)
# remove label in feed
feeds = Collection(db, FEED_REGISTRAR)
item = feeds.find_one(
{'feed_title': feed, '%s.%s' % ('labels', label): {'$exists': True}})
if item:
item['labels'].pop(label)
feeds.update({'_id': item['_id']}, item)
return 'OK'
return None
def update_label(language=None, label=None):
"""
If some feed in label change name, call this one
"""
if not language or not label:
return None
feeds = Collection(db, FEED_REGISTRAR)
# items = feeds.find({'language' = language, '%s.%s' % ('labels', label):
# {'$exists': True}})
# This is incorrect!!!
items = None
if items:
feeds_with_label = [item['feed_title'] for item in items]
if feeds_with_label:
label_name_in_memory = 'news::%s::%s' % (language, label)
# remove all existing data regarding this label in memory
if rclient.exsits(label_name_in_memory):
label_total_in_memory = rclient.zcard(label_name_in_memory)
# remove sequence in label_name_in_memory
rclient.zremrangebyrank(label_name_in_memory, 0,
label_total_in_memory)
# reload entries of feeds to the sequence
ids_all = rclient.keys('*')
for id_in_memory in ids_all:
if 'news' not in id_in_memory and '::' not in id_in_memory:
item = eval(rclient.get(id_in_memory))
if item['feed'] in feeds_with_label:
rclient.zadd(label_name_in_memory, item['updated'],
item['_id'])
return 'OK'
return None
def remove_feed_from_label(language=None, feed=None, label=None):
"""
Remove a feed from a label
"""
if not language or not feed or not label:
return None
# remove member in label sequence in memory regarding the feed
label_name_in_memory = 'news::%s::%s' % (language, label)
if rclient.exists(label_name_in_memory):
label_total_in_memory = rclient.zcard(label_name_in_memory)
ids_with_label = rclient.zrevrange(label_name_in_memory, 0,
label_total_in_memory)
for id_with_label in ids_with_label:
item = eval(rclient.get(id_with_label))
if item['feed'] == feed:
rclient.zrem(id_with_label)
# remove label in feed
feeds = Collection(db, FEED_REGISTRAR)
item = feeds.find_one({
'feed_title': feed,
'%s.%s' % ('labels', label): {
'$exists': True
}
})
if item:
item['labels'].pop(label)
feeds.update({'_id': item['_id']}, item)
return 'OK'
return None
def add_feed_to_label(language=None, feed=None, label=None, label_order=None):
"""
Ad hoc add a feed to label. This mainly deals with history data
"""
if not language or not feed or not label:
return None
# add label to the feed's labels
feeds = Collection(db, FEED_REGISTRAR)
item = feeds.find_one({'feed_title': feed})
if item:
if label not in item['labels']:
item['labels'][label] = label_order
feeds.update({'_id': item['_id']}.item)
# add entries in memory of the feed
label_name_in_memory = 'news::%s::%s' % (language, label)
ids_all = rclient.keys('*')
for id_in_memory in ids_all:
if 'news' not in id_in_memory and '::' not in id_in_memory:
entry = eval(rclient.get(id_in_memory))
if entry['feed'] == feed:
rclient.zadd(label_name_in_memory, entry['updated'],
entry['_id'])
return 'OK'
else:
# label is already there
return None
else:
return None
def clean():
"""
remove expired items from queues in memory
walk through all redis content
"""
logger.info('... cleaning memory ...')
try:
news_lists = rclient.keys('news::*')
for news_list in news_lists:
# get the total number of a news list
news_list_count = rclient.zcard(news_list)
# get all the ids in a news list
if news_list_count:
news_ids = rclient.zrange(news_list, 0, news_list_count)
for news_id in news_ids:
# make sure every item is touched
if not rclient.exists(news_id):
rclient.zrem(news_list, news_id)
else:
news_item_string = rclient.get(news_id)
if news_item_string:
news_item = eval(news_item_string)
news_updated = float(news_item['updated'])
# WTF, remove it
if cleaner.is_overdue(news_updated):
rclient.zrem(news_list, news_id)
rclient.delete(news_id)
else: # check if this is zombie
if _is_zombie(news_item):
rclient.zrem(news_list, news_id)
rclient.delete(news_id)
else:
rclient.zrem(news_list, news_id)
rclient.delete(news_id)
else:
logger.error('Nothing in the list')
continue
return True
except Exception as k:
logger.error(str(k))
return False
def modify_feed(language=None, feed_old=None, feed_new=None):
"""
Call this once a feed's name is changed
"""
if not language or not feed_old or not feed_new:
return None
# update old feed name in memory to new one
feed_old_name_in_memory = 'news::%s::%s' % (language, feed_old)
feed_new_name_in_memory = 'news::%s::%s' % (language, feed_new)
ids_all = rclient.keys('*')
for id_in_memory in ids_all:
if 'news' not in id_in_memory and '::' not in id_in_memory:
entry = eval(rclient.get(id_in_memory))
if entry['feed'] == feed_old:
entry['feed'] = feed_new
rclient.set(name=id_in_memory,
value=entry,
ex=rclient.ttl(id_in_memory),
xx=True)
elif id_in_memory == feed_old_name_in_memory:
rclient.rename(feed_old_name_in_memory, feed_new_name_in_memory)
else:
continue
# update old feed name in database to new one
# change the "meta" data of feed_old in database
feeds = Collection(db, FEED_REGISTRAR)
item = feeds.find_one({'feed_title': feed_old})
if item:
item['feed_title'] = feed_new
feeds.update({'_id': item['_id']}, item)
# change all the related news in database
col = Collection(db, language)
items = col.find({'feed': feed_old})
if items:
for item in items:
item['feed'] = feed_new
col.update({'_id': item['_id']}, item)
return 'OK'
def modify_feed(language=None, feed_old=None, feed_new=None):
"""
Call this once a feed's name is changed
"""
if not language or not feed_old or not feed_new:
return None
# update old feed name in memory to new one
feed_old_name_in_memory = 'news::%s::%s' % (language, feed_old)
feed_new_name_in_memory = 'news::%s::%s' % (language, feed_new)
ids_all = rclient.keys('*')
for id_in_memory in ids_all:
if 'news' not in id_in_memory and '::' not in id_in_memory:
entry = eval(rclient.get(id_in_memory))
if entry['feed'] == feed_old:
entry['feed'] = feed_new
rclient.set(
name=id_in_memory, value=entry,
ex=rclient.ttl(id_in_memory), xx=True)
elif id_in_memory == feed_old_name_in_memory:
rclient.rename(feed_old_name_in_memory, feed_new_name_in_memory)
else:
continue
# update old feed name in database to new one
# change the "meta" data of feed_old in database
feeds = Collection(db, FEED_REGISTRAR)
item = feeds.find_one({'feed_title': feed_old})
if item:
item['feed_title'] = feed_new
feeds.update({'_id': item['_id']}, item)
# change all the related news in database
col = Collection(db, language)
items = col.find({'feed': feed_old})
if items:
for item in items:
item['feed'] = feed_new
col.update({'_id': item['_id']}, item)
return 'OK'
def modify_field(language):
"""
change a key/field name in database and memory
"""
# memory
id_total = rclient.zcard("news::%s" % language)
ids = rclient.zrange("news::%s" % language, 0, id_total)
for id in ids:
entry_string = rclient.get(id)
entry_string_new = entry_string.replace(
'hot_news_image', 'hotnews_image')
rclient.setex(id, rclient.ttl(id), entry_string_new)
# database
col = Collection(db, language)
col.update(
{}, {"rename": {"hot_news_image": "hotnews_image"}}, False, True)
col.update(
{}, {"rename": {"hot_news_image_local": "hotnews_image_local"}}, False,
True)
def modify_field(language):
"""
change a key/field name in database and memory
"""
# memory
id_total = rclient.zcard("news::%s" % language)
ids = rclient.zrange("news::%s" % language, 0, id_total)
for id in ids:
entry_string = rclient.get(id)
entry_string_new = entry_string.replace('hot_news_image',
'hotnews_image')
rclient.setex(id, rclient.ttl(id), entry_string_new)
# database
col = Collection(db, language)
col.update({}, {"rename": {
"hot_news_image": "hotnews_image"
}}, False, True)
col.update({}, {"rename": {
"hot_news_image_local": "hotnews_image_local"
}}, False, True)