def Fetch(config_name,redis):
config = dict()
for key in 'user','experiment','sources':
if redis.type(config_name+':'+key) == 'hash':
config[key] = redis.hgetall(config_name+':'+key)
if redis.type(config_name+':'+key) == 'set':
config[key] = dict()
for item in redis.smembers(config_name+':'+key):
config[key][item] = redis.hgetall('source:'+config_name+':'+item)
return config
def repr(key):
key_type = redis.type(key)
strategies = {
'string': 'repr_string'
, 'list': 'repr_list'
, 'set': 'repr_set'
, 'zset': 'repr_zset'
, 'hash': 'repr_hash'
, 'none': 'repr_none'
}
return {
'key': key
, 'repr': strategy ( strategies, key_type, key )
, 'type': redis.type(key)
}
def hashes_redis():
import redis
redis = redis.Redis(host='127.0.0.1',
port=6379,
db=0,
charset="utf-8",
decode_responses=True)
print("-------------")
print("HASH")
print("-------------")
#hmset(name, mapping) hget(name, key) hgetall(name)
redis.hmset("user.1", {"name": "peter", "email": "*****@*****.**"})
print("map.1: ", redis.hgetall("user.1"))
print("name.1:", redis.hget("user.1", "name"))
print("email.1:", redis.hget("user.1", "email"))
print("-------------")
#hset(key, field, value) hget()
redis.hset("user.2", "name1", "peter")
print("map.2: ", redis.hgetall("user.2"))
print("type map.2: ", redis.type("user.2"))
print("name.2:", redis.hget("user.2", "name1"))
print("-------------")
#delete all keys
for key in redis.scan_iter("prefix:*"):
redis.delete(key)
def show_key(key):
return json.dumps ( definite_strategy (
{
'string': lambda x: redis.get(x)
, 'hash': lambda x: redis.hgetall(x)
, 'list': lambda x: redis.lrange(x,0,-1)
}
, redis.type(key)
, key
) )
def dump_all(redis=r):
keys = redis.keys('*')
pairs = {}
for key in keys:
type = redis.type(key)
val = redis.get(key)
try:
pairs[key] = eval(val)
except Exception as e:
print pairs, key, val, e
return pairs
def dump_all(redis=r):
keys = redis.keys('*')
pairs = {}
for key in keys:
type = redis.type(key)
val = redis.get(key)
try:
pairs[key] = eval(val)
except:
pass
return pairs
def f(key):
return ( definite_strategy (
{ 'list': lambda x: reduce(lambda x,y: x+y, [ f(a) if a[0:2] == '::' or a[0:2] == ':#' else [a] for a in redis.lrange(x, 0, -1) ])
, 'hash': lambda x: [redis.hgetall(x)]
, 'none': lambda x: []
, 'string': lambda x: [ f(a) if a[0:2] == '::' else a for a in [redis.get(x)] ]
}
, redis.type(key)
, key
, default_strategy = 'hash'
)
)
def render(key):
# print 'rendering ' + key
key_type = redis.type(key)
strategies = {
'string': 'repr_string'
, 'list': 'render_list'
, 'set': 'repr_set'
, 'zset': 'repr_zset'
, 'hash': 'render_hash'
, 'none': 'repr_none'
}
return strategy ( strategies, key_type, key )
def clear_sub_nodes(type_name, hashcode):
namehash = "%s:%s" % (type_name, hashcode)
metahash = "%s:linked_fields:%s" % (type_name, hashcode)
m_type = redis.type(metahash)
if m_type == "list":
for index in redis.lrange(metahash, 0, -1):
i = int(index)
clear_sub_nodes(type_name, redis.lindex(namehash, i))
elif m_type == "hash":
meta_dictionary = redis.hgetall(namehash)
for key in meta_dictionary:
linked_node_hash = meta_dictionary[key]
clear_sub_nodes(type_name, linked_node_hash)
redis.delete(namehash)
redis.delete(metahash)
def record(self, redis, key):
debug_info = redis.debug_object(key)
object_type = redis.type(key)
if self.type is None:
self.type = object_type
elif self.type != object_type:
self.type = 'multiple'
else: pass
if self.type == 'zset':
l = redis.zcard(key)
self.item_lengths.append(l)
self.count += 1
self.bytes += debug_info['serializedlength']
def record(self, redis, key):
debug_info = redis.debug_object(key)
object_type = redis.type(key)
if self.type is None:
self.type = object_type
elif self.type != object_type:
self.type = 'multiple'
else:
pass
if self.type == 'zset':
l = redis.zcard(key)
self.item_lengths.append(l)
self.count += 1
self.bytes += debug_info['serializedlength']
filter = '*pravda*'
filter = '*nos*html'
filter = '*dw.de*'
filter = "*sixth-division-coach*"
keys = redis.keys(filter)
for key in keys:
id = id + 1
datasetfile = DATASETDIR + "/sample" + str(id) + ".txt"
contentfile = CONTENTDIR + "/sample" + str(id) + ".txt"
origfile = ORIGDIR + "/sample" + str(id) + ".txt"
newsfile = NEWSDIR + "/sample" + str(id) + ".txt"
content = open(contentfile, 'w')
dataset = open(datasetfile, "w")
origin = open(origfile, "w")
news = open(newsfile, "w")
type = redis.type(key)
value = ''
if type == 'string':
val = redis.get(key)
html = val
if count <= limit:
print key
content.write(key + '\n')
dataset.write(HEADLINE + '\n')
result = chardet.detect(html)
charset = result['encoding']
if charset == 'utf-8':
data = html
else:
data = html.decode(charset)
redis = redis.Redis(host='localhost', port=6379, db=0)
# DEBUG levels
# extractor
# distance
# cluster
# generic
DEBUG = 'clusters'
#DEBUG = 'distance'
filter = "*2015/02/13/7058423*"
keys = redis.keys(filter)
id = 0
for key in keys:
id = id + 1
type = redis.type(key);
value = ''
if type == 'string':
val = redis.get(key);
html = val
result = chardet.detect(html)
charset = result['encoding']
if charset == 'utf-8':
data = html
else:
data = html.decode(charset)
try:
(x, y, doc, freq, posindex, mainindex) = buildpattern(data, 'd')
def rewrite(key):
if redis.type(key) == 'list':
del_key_f(key)
return add_key_f(key)
