def clear_cache(cls): cache_key = '%s*' %TAG_CACHE_KEY keys = cache.keys(cache_key) for key in keys: cache.delete(key) print("C ache clear Done")
def to_file(cls, path=path): import codecs file = codecs.open(path, 'w+', 'utf-8') keys = cache.keys('%s*' %TAG_CACHE_KEY) for key in keys: name = key[4:] file.write('%s\n' %name.decode('utf-8')) file.close()
def to_index(cls): keys = cache.keys(pattern='%s*' %BASIC_TAG_PREFIX) for key in keys: cache.delete(key) items = cls.objects() for item in items: if len(item.name) > 1: key = '%s%s' %(BASIC_TAG_PREFIX, item.name) cache.set(name=key, value=item.score)
def loads_from_pymongo(cls): tem_key = 'TEMP:KEY:' items = db.tags.find() for item in items: for tag in item['tags']: print("incr tag name %s" %tag) key = tem_key + tag cache.incr(name=key, amount=1) keys = cache.keys(pattern='%s*' %tem_key) for key in keys: name = key[9:].decode('utf-8') score = float(cache.get(key)) cls.create_tag(name=name, score=score) cache.delete(key)
def loads_basic_tags_to_cache(cls): from pymongo import Connection c = Connection() db = c.names tags = db.results.find() keys = cache.keys(pattern='%s*' %BASIC_TAG_PREFIX) for key in keys: cache.delete(key) for tag in tags: name = tag['name'].split(':')[1] key = BASIC_TAG_PREFIX + name value = tag['value'] if int(value) > 1: cache.set(key, name)
def see_top_tags(cls): keys = cache.keys(pattern='%s*' %BASIC_CACHE_PREFIX) set_name = 'TEMP:SET' for key in keys: value = float(cache.get(key)) key = key[9:] cache.zadd(set_name, value, key) results = cache.zrevrange(name=set_name, start=0, num=LIMIT, withscores=True) results = [{'name': r[0].decode('utf-8'), 'value': r[1]} for r in results] from pymongo import Connection c = Connection() db = c.names if results: db.results.insert(results) cache.delete(set_name) return results
def handle(self, *args, **options): path = os.getcwd() + "/data/" # files = os.listdir(path) # # for file_name in files: # file = open(path + file_name, 'r') # lines = file.readlines() # for line in lines: # try: # load = False # line.replace('::::::', ':::') # line.replace(':::::', ':::') # line.replace('::::', ':::') # tags_str = line.split(':::')[3] # tags = tags_str.split('__') # for tag in tags: # if cache.exists(name='%s%s' %(TMP_KEY, tag)): # load = True # break # # if load: # tag = tags[0] # for last in tags[1:]: # key1 = '%s%s' %(BASIC_KEY, tag) # key2 = '%s%s' %(BASIC_KEY, last) # cache.zincrby(name=key1, value=last, amount=damping) # cache.zincrby(name=key2, value=tag, amount=damping) # # except Exception as err: # print(err) # continue path = path + "basic_tags/" keys = cache.keys("%s*" % BASIC_KEY) file = open(path + "relations", "w") for key in keys: print("load %s" % key[15:]) data = cache.zrevrangebyscore(name=key, max="+inf", min="-inf", withscores=True) name = key[15:] values = "__".join([l[0] + ":::" + str(l[1]) for l in data]) file.write(name + "\t" + values + "\n") file.close()
def create_action(self): cache_key = "WEIBO:HOT:%s" %self.user.sns_id cache.delete(cache_key) tmp_cache_key = "TEMP:WEIBO:HISTORY:%s:::" %self.user.sns_id weibo_history = self.user.weibo_history for text in weibo_history: terms = seg_txt(text.encode('utf-8')) for term in terms: index_key = '%s%s' %(BASIC_TAG_PREFIX, term) if cache.exists(index_key): key = tmp_cache_key + term.decode('utf-8') cache.incr(name=key, amount=1) keys = cache.keys(pattern="%s*" %tmp_cache_key) for key in keys: name = key.split(":::")[1] value = float(cache.get(key)) cache.zadd(cache_key, value, name) cache.delete(key) tag = BasicTag.get_by_name(name=name) if not tag: continue relations = tag.friends score = tag.score for f in relations: items = f.split(':::') obj_name = items[0] obj_value = float(items[1]) result = obj_value/50*value cache.zadd(cache_key, result, obj_name) results = cache.zrevrange(name=cache_key, start=0, num=30, withscores=True) tags = [result[0].decode('utf-8') +'__' + str(result[1]) for result in results] self.user.update(set__tags=tags)
def calculate_relations(cls): items = cls.objects(friends__size=0) for item in items: try: name = item.name tag = db.tags.find_one({'name': name}) except: continue if not tag: continue item_tags = tag['tags'] for item_tag in item_tags: try: cls.__calculate_relation(obj_tag=item_tag, item_name=name) except: continue keys = cache.keys('TEMP:FRIENDS:%s*' %item_tag) for key in keys: name = key.split(':::')[1] cache.zadd('TEMP:%s' %item_tag, float(cache.get(key)), name) cache.delete(key) results = cache.zrevrange(name='TEMP:%s' %item_tag, start=0, num=DEFAULT_LIMIT, withscores=True) cache.delete('TEMP:%s' %item_tag) for r in results: name = r[0].decode('utf-8') score = r[1]/1000 if name == item_tag: continue try: obj = cls.objects.get(name=item_tag) obj.update(add_to_set__friends='%s:::%f' %(name, score)) print("push %s:::%f to tag %s" %(name, score, item_tag)) except : cls.create_tag(name=item_tag, friends=['%s:::%f' %(name, score),]) print("push %s:::%f to create tag %s" %(name, score, item_tag))
def delete_keys(): keys = cache.keys(pattern='%s*' %BASIC_CACHE_PREFIX) print("keys count %d" %len(keys)) for key in keys: cache.delete(key)
def clear_cache(cls): keys = cache.keys('%s*' %cache_key) for key in keys: cache.delete(key) print("Delete Done")