class IndexCache(object):
def __init__(self, host, port):
self.client = StrictRedis(host=host, port=port, db=0)
def build(self, doc_word_scores):
"""Clears the entire store and adds all the doc_word_scores into a
hash table in the store.
:param doc_word_scores: dictionary of dictionaries that looks like:
doc_word_scores[word][doc_id] = score of word in that
document
"""
self.reset()
for word, doc_id_score in doc_word_scores.items():
# Add table name to word
word_key = DOCUMENT_WORD_SCORE_NAME + word
self.client.hmset(word_key, doc_id_score)
self.save_to_disk()
def reset(self):
"""Deletes all keys in this DB.
"""
self.client.flushdb()
def save_to_disk(self):
"""Asyncronous write to disk for persistent storage.
"""
self.client.bgsave()
def to_dict(self):
"""Returns the "doc_word_scores" table in Redis in dictionary form.
"""
doc_word_scores = {}
for word_key in self.doc_word_scores_iter():
# Remove the table name from the key
word = word_key.replace(DOCUMENT_WORD_SCORE_NAME, "")
# Grab the {doc_ids : scores} dictionary for word
doc_word_scores[word] = self.client.hgetall(word_key)
return doc_word_scores
def doc_word_scores_iter(self):
"""Returns an iterator for the keys of all the words stored in Redis
"""
return self.client.scan_iter(match=DOCUMENT_WORD_SCORE_NAME + "*")
def is_empty(self):
return self.client.dbsize() <= 0
def doc_scores(self, word):
"""Returns a hash table of document_ids mapping to scores
"""
word_key = DOCUMENT_WORD_SCORE_NAME + word
return self.client.hgetall(word_key)
def string():
redis = StrictRedis(host='localhost', port=6379, password='', db=0)
# result=redis.set('name','zengshuai')
# print(result)
# redis.set('age',25)
# redis.set('gender','male')
# redis.set('score',100)
#通用方法
print(redis.exists('name'))
print(redis.type('name'))
print(redis.keys('a*')) #a 开头的 键值
print(redis.randomkey()) #随机取一个键值
# redis.rename('score','English') #重命名
# print(redis.get('English'))
print(redis.dbsize()) #size
redis.expire('English', 2) #设置键值过期时间
print(redis.ttl('English')) #获取键值过期时间
redis.move('age', 1) #将键移动到其他数据库
# redis.flushdb() #清空本数据库
# redis.flushall() #清空所有数据库
#字符串操作
redis.getset('name', 'kate')
print(redis.get('name'))
def check_redis(ip):
try:
redis = StrictRedis(host=ip, port=6379, db=0, password='')
redis.set('name', 'BKB')
print(redis.get('name'))
print(redis.dbsize())
click.secho('[+] 存在reids未授权:', fg='red')
except:
print('[-] -> 不存在reids未授权')
def check_redis(ip):
try:
redis = StrictRedis(host=ip,port=6379,db=0,password='')
if(redis.dbsize()):
return True
else:
return False
except:
return False
def find_empty_redis_database():
"""Tries to connect to a random Redis database (starting from 4), and
will use/connect it when no keys are in there.
"""
for dbnum in range(4, 17):
testconn = StrictRedis(db=dbnum)
empty = testconn.dbsize() == 0
if empty:
return testconn
assert False, 'No empty Redis database found to run tests in.'
def find_empty_redis_database():
"""Tries to connect to a random Redis database (starting from 4), and
will use/connect it when no keys are in there.
"""
for dbnum in range(4, 17):
testconn = StrictRedis(db=dbnum)
empty = testconn.dbsize() == 0
if empty:
return testconn
assert False, 'No empty Redis database found to run tests in.'
class redisUtil:
def __init__(self):
self.redis = StrictRedis(host='localhost', port=6379, db=0)
def setValue(self, key, value):
self.redis.set(key, value)
def getValue(self, key):
result = self.redis.get(key)
if (result == None):
return result
return self.redis.get(key).decode()
def getDbSize(self):
return self.redis.dbsize()
def updateAndReturn(self, key, value):
return self.redis.getset(key, value)
def delete(self, key):
return self.redis.delete(key)
class GraphStorage(object):
def __init__(self, link_file='links.txt', host='localhost', port=6379):
self.db = StrictRedis(host, port)
self.link_file = link_file
def add_nodes(self, graphDict, nameDict):
pipe = self.db.pipeline()
with open(self.link_file) as link_file:
for _ in xrange(1):
for _ in xrange(570607):
node, links = self.create_link(next(link_file))
name = linecache.getline('titles.txt', node)
pipe.rpush('links-{0}'.format(node), *links)
pipe.append('name-{0}'.format(node), name)
pipe.append('id-{0}'.format(name), node)
pipe.execute()
def create_link(self, link):
start, targets = link.rstrip('\n').split(': ')
return int(start), map(int, targets.split())
def get(self, kind, value):
key = '{0}-{1}'.format(kind, value)
if kind in ('name', 'id'):
return self.db.get(key)
elif kind == 'links':
return self.db.lrange(key, 0, -1)
raise ValueError
def is_leaf(self, value):
return self.db.sismember('leaves', value)
def flush(self):
self.db.flushall()
def __len__(self):
return self.db.dbsize()
# 5706070
class RedisCache:
def __init__(self, client = None, encoding = 'utf-8', db = 0, compress = True):
# if a client object is not passed then try
# connecting to redis at the default localhost port
self.client = StrictRedis(host = 'localhost', port = 6379, db = db) if client is None else client
# self.expires = expires
self.encoding = encoding
self.compress = compress
def __getitem__(self, url):
'''
Load value from Redis for the given URL
'''
record = self.client.get(url)
if record:
if self.compress:
record = zlib.decompress(record)
try:
rec = record.decode(self.encoding)
except UnicodeDecodeError:
rec = bytes(json.dumps({'html' : None, 'code' : 403}), self.encoding)
return json.loads(rec)
else:
raise KeyError(url + ' does not exist.')
def __setitem__(self, url, result):
'''
Save value in Redis for the given URL
'''
data = bytes(json.dumps(result), self.encoding, errors = 'ignore')
if self.compress:
data = zlib.compress(data)
self.client.set(url, data)
def __len__(self):
return self.client.dbsize()
def erase(self):
self.client.flushdb()
# 通过正则获取所有符合类型的 key, 不过这里也不是什么普通正则规范, .* 貌似就不好使
print(redis.keys('n*'))
# 获取随机的一个值
print(redis.randomkey())
# key 重命名
redis.rename('name','nikename')
# 获取当前数据库 key 的数量
print(redis.dbsize())
# 设置 key 的过期时间 单位为秒
redis.expire('name', 2)
# 获取 key 的过期时间. 单位为秒 -1 代表永不过期
print(redis.ttl('name'))
# 移动key 到其他的数据库
redis.move('name',2) # 2为数据库代号
# 删除当前选择数据库数据
#Author guo
from redis import StrictRedis
redis = StrictRedis(host='127.0.0.1', port=6379, db=0,
password='******') #或者可以使用连接pool的方法进行连接
redis.set('name', 'Bob') #调用set方法，设置一个键值对
print(redis.get('name')) #获取并打印
redis.set('age', 22)
print(redis.get('age'))
print(redis.exists('name')) #是否存在键名
print(redis.type('name')) #键的类型b'string'
print(redis.keys('n*')) #获取所有以n开头的键[b'name']
print(redis.randomkey()) #随机获取一个键
redis.rename('name', 'Name')
print(redis.dbsize()) #获取当前数据库大小
#redis.expire('name',2) 将name的过期时间设置为2s
print(redis.ttl('name')) #获取过期时间
#move（name,db）移动到别的数据库
#flushdb（）删除当前选择数据库中的所有键
#flushall（） 删除所有数据库中所有的键
# redis数据库检测
from redis import StrictRedis
red = StrictRedis(host='localhost', port=6379, db=0, password='******')
# red.set('name', 'Bob')
# print(red.get('name'))
# print(red.exists('mykey1'))
print(red.dbsize())
from redis import StrictRedis, ConnectionPool
redis = StrictRedis(host='localhost', port=6379, db=0,
password=None) #两种连接方式都可以
# pool=ConnectionPool(host='localhost',port=6379,db=0,password=None)
# redis=StrictRedis(connection_pool=pool)
redis.set('name', 'job')
redis.set('age', 22)
print(redis.get('name'))
print(redis.exists('name'))
redis.delete('age')
print(type('name'))
print(redis.keys('n*'))
print(redis.randomkey())
redis.move('name', 2)
# redis.flushdb() #删除当前选择数据库中的所有的键
# redis.flushall() #删除所有数据库中的所有的键
print(redis.dbsize()) #获取当前数据库中的所有键的数目
class database():
'''
Further Encapsulate Redis
'''
def __init__(self,
host=default_database_host,
port=default_database_port,
db=0,
password=default_database_password):
if default_database_type.lower() == 'redis':
self.__redis = StrictRedis(host=host,
port=port,
db=db,
password=password)
if default_database_flushall:
self.delete_all() #WARNING
else:
raise ToolBox.ConfigError('Only support Redis database now !')
def get_num(self):
'''
:return:int, num of keys in database
'''
size = self.__redis.dbsize()
if size:
return self.__redis.dbsize()
else:
return 0
def delete_all(self):
self.__redis.flushdb()
print(f'Redis has been flushed all {ToolBox.tool_get_current_time()}')
def add_proxies(self, proxies):
if proxies:
data = {}
for proxy in proxies:
string_proxy = proxy.get_string_address()
dict_proxy = json.dumps(proxy.get_dict_address())
data[string_proxy] = dict_proxy
self.__redis.mset(data)
# print(f'Redis num {self.get_num()}')
def get_one_string(self) -> str or None:
'''
:return:string, proxy
'''
if self.get_num():
byte_ = self.__redis.randomkey()
str_ = str(byte_, encoding='utf-8')
return str_
else:
return None
def get_one_dict(self):
'''
:return:dictionary, proxy
'''
if self.get_num():
dict_proxy = self.__redis.get(self.__redis.randomkey())
dict_proxy = json.loads(dict_proxy)
return dict_proxy
else:
return None
def delete_proxies(self, proxies):
'''
delete the proxies from database
:param proxies: proxy object list
:return: None
'''
if proxies:
for proxy in proxies:
self.__redis.delete(proxy.get_string_address())
# print(f'Redis num {self.get_num()}')
def proxy_feedback(self, proxy_string_dict, flag: bool):
'''
through this func to adjust proxy points
when you get a proxy from database
if flag == False, then the proxy's points will be decreased
== True, then the proxy's points will be increased
:return: insert a special message to the database
'''
res = proxy_string_dict
if isinstance(proxy_string_dict, dict):
res = proxy_string_dict['http']
if flag:
self.__redis.rpush('increase', res)
else:
self.__redis.rpush('decrease', res)
def get_feedback(self) -> dict:
feedback = {}
info = []
temp = self.__redis.rpop('increase')
while temp:
temp = bytes.decode(temp)
info.append(temp)
temp = self.__redis.rpop('increase')
feedback['increase'] = info
info = []
temp = self.__redis.rpop('decrease')
while temp:
temp = bytes.decode(temp)
info.append(temp)
temp = self.__redis.rpop('decrease')
feedback['decrease'] = info
return feedback
print(redis.get('name'))
"""键操作"""
# 判断是否存在
redis.exists('name')
# 删除
# redis.delete('name')
# 获取键的类型
redis.type('name')
# 正则匹配
redis.keys('n*')
# 随机一个键
redis.randomkey()
# 重命名
redis.rename('name', 'nickname')
# 获取键的数目
redis.dbsize()
# 设置键的过期时间 2秒
redis.expire('name', 2)
# 获取键的过期时间 -1表示永远不过期
redis.ttl('name')
# 将键移动到其他数据库 移动到2号数据库
redis.move('name', 2)
# 删除当前选择数据库中所有的键
redis.flushdb()
# 删除所有数据库中的所有键
redis.flushall()
"""字符串操作"""
# 设值
redis.set('age', 20)
# 取值
redis.get('age')
