def Fetch(config_name,redis): config = dict() for key in 'user','experiment','sources': if redis.type(config_name+':'+key) == 'hash': config[key] = redis.hgetall(config_name+':'+key) if redis.type(config_name+':'+key) == 'set': config[key] = dict() for item in redis.smembers(config_name+':'+key): config[key][item] = redis.hgetall('source:'+config_name+':'+item) return config
def repr(key): key_type = redis.type(key) strategies = { 'string': 'repr_string' , 'list': 'repr_list' , 'set': 'repr_set' , 'zset': 'repr_zset' , 'hash': 'repr_hash' , 'none': 'repr_none' } return { 'key': key , 'repr': strategy ( strategies, key_type, key ) , 'type': redis.type(key) }
def hashes_redis(): import redis redis = redis.Redis(host='127.0.0.1', port=6379, db=0, charset="utf-8", decode_responses=True) print("-------------") print("HASH") print("-------------") #hmset(name, mapping) hget(name, key) hgetall(name) redis.hmset("user.1", {"name": "peter", "email": "*****@*****.**"}) print("map.1: ", redis.hgetall("user.1")) print("name.1:", redis.hget("user.1", "name")) print("email.1:", redis.hget("user.1", "email")) print("-------------") #hset(key, field, value) hget() redis.hset("user.2", "name1", "peter") print("map.2: ", redis.hgetall("user.2")) print("type map.2: ", redis.type("user.2")) print("name.2:", redis.hget("user.2", "name1")) print("-------------") #delete all keys for key in redis.scan_iter("prefix:*"): redis.delete(key)
def show_key(key): return json.dumps ( definite_strategy ( { 'string': lambda x: redis.get(x) , 'hash': lambda x: redis.hgetall(x) , 'list': lambda x: redis.lrange(x,0,-1) } , redis.type(key) , key ) )
def dump_all(redis=r): keys = redis.keys('*') pairs = {} for key in keys: type = redis.type(key) val = redis.get(key) try: pairs[key] = eval(val) except Exception as e: print pairs, key, val, e return pairs
def dump_all(redis=r): keys = redis.keys('*') pairs = {} for key in keys: type = redis.type(key) val = redis.get(key) try: pairs[key] = eval(val) except: pass return pairs
def f(key): return ( definite_strategy ( { 'list': lambda x: reduce(lambda x,y: x+y, [ f(a) if a[0:2] == '::' or a[0:2] == ':#' else [a] for a in redis.lrange(x, 0, -1) ]) , 'hash': lambda x: [redis.hgetall(x)] , 'none': lambda x: [] , 'string': lambda x: [ f(a) if a[0:2] == '::' else a for a in [redis.get(x)] ] } , redis.type(key) , key , default_strategy = 'hash' ) )
def render(key): # print 'rendering ' + key key_type = redis.type(key) strategies = { 'string': 'repr_string' , 'list': 'render_list' , 'set': 'repr_set' , 'zset': 'repr_zset' , 'hash': 'render_hash' , 'none': 'repr_none' } return strategy ( strategies, key_type, key )
def clear_sub_nodes(type_name, hashcode): namehash = "%s:%s" % (type_name, hashcode) metahash = "%s:linked_fields:%s" % (type_name, hashcode) m_type = redis.type(metahash) if m_type == "list": for index in redis.lrange(metahash, 0, -1): i = int(index) clear_sub_nodes(type_name, redis.lindex(namehash, i)) elif m_type == "hash": meta_dictionary = redis.hgetall(namehash) for key in meta_dictionary: linked_node_hash = meta_dictionary[key] clear_sub_nodes(type_name, linked_node_hash) redis.delete(namehash) redis.delete(metahash)
def record(self, redis, key): debug_info = redis.debug_object(key) object_type = redis.type(key) if self.type is None: self.type = object_type elif self.type != object_type: self.type = 'multiple' else: pass if self.type == 'zset': l = redis.zcard(key) self.item_lengths.append(l) self.count += 1 self.bytes += debug_info['serializedlength']
filter = '*pravda*' filter = '*nos*html' filter = '*dw.de*' filter = "*sixth-division-coach*" keys = redis.keys(filter) for key in keys: id = id + 1 datasetfile = DATASETDIR + "/sample" + str(id) + ".txt" contentfile = CONTENTDIR + "/sample" + str(id) + ".txt" origfile = ORIGDIR + "/sample" + str(id) + ".txt" newsfile = NEWSDIR + "/sample" + str(id) + ".txt" content = open(contentfile, 'w') dataset = open(datasetfile, "w") origin = open(origfile, "w") news = open(newsfile, "w") type = redis.type(key) value = '' if type == 'string': val = redis.get(key) html = val if count <= limit: print key content.write(key + '\n') dataset.write(HEADLINE + '\n') result = chardet.detect(html) charset = result['encoding'] if charset == 'utf-8': data = html else: data = html.decode(charset)
redis = redis.Redis(host='localhost', port=6379, db=0) # DEBUG levels # extractor # distance # cluster # generic DEBUG = 'clusters' #DEBUG = 'distance' filter = "*2015/02/13/7058423*" keys = redis.keys(filter) id = 0 for key in keys: id = id + 1 type = redis.type(key); value = '' if type == 'string': val = redis.get(key); html = val result = chardet.detect(html) charset = result['encoding'] if charset == 'utf-8': data = html else: data = html.decode(charset) try: (x, y, doc, freq, posindex, mainindex) = buildpattern(data, 'd')
def rewrite(key): if redis.type(key) == 'list': del_key_f(key) return add_key_f(key)