def clear_cache(cls):
cache_key = '%s*' %TAG_CACHE_KEY
keys = cache.keys(cache_key)
for key in keys:
cache.delete(key)
print("C ache clear Done")
def to_file(cls, path=path):
import codecs
file = codecs.open(path, 'w+', 'utf-8')
keys = cache.keys('%s*' %TAG_CACHE_KEY)
for key in keys:
name = key[4:]
file.write('%s\n' %name.decode('utf-8'))
file.close()
def to_index(cls):
keys = cache.keys(pattern='%s*' %BASIC_TAG_PREFIX)
for key in keys:
cache.delete(key)
items = cls.objects()
for item in items:
if len(item.name) > 1:
key = '%s%s' %(BASIC_TAG_PREFIX, item.name)
cache.set(name=key, value=item.score)
def loads_from_pymongo(cls):
tem_key = 'TEMP:KEY:'
items = db.tags.find()
for item in items:
for tag in item['tags']:
print("incr tag name %s" %tag)
key = tem_key + tag
cache.incr(name=key, amount=1)
keys = cache.keys(pattern='%s*' %tem_key)
for key in keys:
name = key[9:].decode('utf-8')
score = float(cache.get(key))
cls.create_tag(name=name, score=score)
cache.delete(key)
def loads_basic_tags_to_cache(cls):
from pymongo import Connection
c = Connection()
db = c.names
tags = db.results.find()
keys = cache.keys(pattern='%s*' %BASIC_TAG_PREFIX)
for key in keys:
cache.delete(key)
for tag in tags:
name = tag['name'].split(':')[1]
key = BASIC_TAG_PREFIX + name
value = tag['value']
if int(value) > 1:
cache.set(key, name)
def see_top_tags(cls):
keys = cache.keys(pattern='%s*' %BASIC_CACHE_PREFIX)
set_name = 'TEMP:SET'
for key in keys:
value = float(cache.get(key))
key = key[9:]
cache.zadd(set_name, value, key)
results = cache.zrevrange(name=set_name, start=0, num=LIMIT, withscores=True)
results = [{'name': r[0].decode('utf-8'), 'value': r[1]} for r in results]
from pymongo import Connection
c = Connection()
db = c.names
if results:
db.results.insert(results)
cache.delete(set_name)
return results
def handle(self, *args, **options):
path = os.getcwd() + "/data/"
# files = os.listdir(path)
#
# for file_name in files:
# file = open(path + file_name, 'r')
# lines = file.readlines()
# for line in lines:
# try:
# load = False
# line.replace('::::::', ':::')
# line.replace(':::::', ':::')
# line.replace('::::', ':::')
# tags_str = line.split(':::')[3]
# tags = tags_str.split('__')
# for tag in tags:
# if cache.exists(name='%s%s' %(TMP_KEY, tag)):
# load = True
# break
#
# if load:
# tag = tags[0]
# for last in tags[1:]:
# key1 = '%s%s' %(BASIC_KEY, tag)
# key2 = '%s%s' %(BASIC_KEY, last)
# cache.zincrby(name=key1, value=last, amount=damping)
# cache.zincrby(name=key2, value=tag, amount=damping)
#
# except Exception as err:
# print(err)
# continue
path = path + "basic_tags/"
keys = cache.keys("%s*" % BASIC_KEY)
file = open(path + "relations", "w")
for key in keys:
print("load %s" % key[15:])
data = cache.zrevrangebyscore(name=key, max="+inf", min="-inf", withscores=True)
name = key[15:]
values = "__".join([l[0] + ":::" + str(l[1]) for l in data])
file.write(name + "\t" + values + "\n")
file.close()
def create_action(self):
cache_key = "WEIBO:HOT:%s" %self.user.sns_id
cache.delete(cache_key)
tmp_cache_key = "TEMP:WEIBO:HISTORY:%s:::" %self.user.sns_id
weibo_history = self.user.weibo_history
for text in weibo_history:
terms = seg_txt(text.encode('utf-8'))
for term in terms:
index_key = '%s%s' %(BASIC_TAG_PREFIX, term)
if cache.exists(index_key):
key = tmp_cache_key + term.decode('utf-8')
cache.incr(name=key, amount=1)
keys = cache.keys(pattern="%s*" %tmp_cache_key)
for key in keys:
name = key.split(":::")[1]
value = float(cache.get(key))
cache.zadd(cache_key, value, name)
cache.delete(key)
tag = BasicTag.get_by_name(name=name)
if not tag:
continue
relations = tag.friends
score = tag.score
for f in relations:
items = f.split(':::')
obj_name = items[0]
obj_value = float(items[1])
result = obj_value/50*value
cache.zadd(cache_key, result, obj_name)
results = cache.zrevrange(name=cache_key, start=0, num=30, withscores=True)
tags = [result[0].decode('utf-8') +'__' + str(result[1]) for result in results]
self.user.update(set__tags=tags)
def calculate_relations(cls):
items = cls.objects(friends__size=0)
for item in items:
try:
name = item.name
tag = db.tags.find_one({'name': name})
except:
continue
if not tag:
continue
item_tags = tag['tags']
for item_tag in item_tags:
try:
cls.__calculate_relation(obj_tag=item_tag, item_name=name)
except:
continue
keys = cache.keys('TEMP:FRIENDS:%s*' %item_tag)
for key in keys:
name = key.split(':::')[1]
cache.zadd('TEMP:%s' %item_tag, float(cache.get(key)), name)
cache.delete(key)
results = cache.zrevrange(name='TEMP:%s' %item_tag, start=0, num=DEFAULT_LIMIT, withscores=True)
cache.delete('TEMP:%s' %item_tag)
for r in results:
name = r[0].decode('utf-8')
score = r[1]/1000
if name == item_tag:
continue
try:
obj = cls.objects.get(name=item_tag)
obj.update(add_to_set__friends='%s:::%f' %(name, score))
print("push %s:::%f to tag %s" %(name, score, item_tag))
except :
cls.create_tag(name=item_tag, friends=['%s:::%f' %(name, score),])
print("push %s:::%f to create tag %s" %(name, score, item_tag))
def delete_keys():
keys = cache.keys(pattern='%s*' %BASIC_CACHE_PREFIX)
print("keys count %d" %len(keys))
for key in keys:
cache.delete(key)
def clear_cache(cls):
keys = cache.keys('%s*' %cache_key)
for key in keys:
cache.delete(key)
print("Delete Done")
