class load_class:
def __init__(self):
self.rc = StrictRedisCluster(startup_nodes=config.startup_nodes)
self.key_num = 1
def _string(self, num, string_value_len, thread_num):
key_num = self.key_num
while key_num <= num:
key_name = 'string_r_test_' + str(thread_num) + ':00' + str(
key_num)
self.rc.set(
key_name, ''.join(
map(lambda xx: (hex(ord(xx))[2:]),
os.urandom(int(string_value_len) / 2))))
key_num += 1
def _hash(self, num, hash_value_len, thread_num, filed_num):
key_num = self.key_num
while key_num <= num:
key_name = 'hash_r_test_' + str(thread_num) + ':00' + str(key_num)
_file_num = 1
filed_set = {}
while _file_num <= filed_num:
key_filed = str(_file_num) + ':value_test'
filed_set[key_filed] = ''.join(
map(lambda xx: (hex(ord(xx))[2:]),
os.urandom(hash_value_len / 2)))
_file_num += 1
self.rc.hmset(key_name, filed_set)
key_num += 1
def _set(self, num, set_value_len, thread_num, value_num):
key_num = self.key_num
while key_num <= num:
key_name = 'set_r_test_' + str(thread_num) + ':00' + str(key_num)
_value_num = 1
value_set = []
while _value_num <= value_num:
self.rc.sadd(
key_name, ''.join(
map(lambda xx: (hex(ord(xx))[2:]),
os.urandom(set_value_len / 2))))
_value_num += 1
key_num += 1
def _list(self, num, list_value_len, thread_num, value_num):
key_num = self.key_num
while key_num <= num:
key_name = 'list_r_test_' + str(thread_num) + ':00' + str(key_num)
_value_num = 1
value_set = []
while _value_num <= value_num:
self.rc.lpush(
key_name, ''.join(
map(lambda xx: (hex(ord(xx))[2:]),
os.urandom(list_value_len / 2))))
_value_num += 1
key_num += 1
def load(cluster, datapaths):
print datapaths, "process start"
rc = StrictRedisCluster(startup_nodes=[REDIS_CONFIG.get(cluster)],
decode_responses=True)
with open(datapaths, 'rb') as csv_file:
data_reader = csv.reader(csv_file, delimiter="\t")
for row in data_reader:
rc.hmset(row[0], {'a': row[1], 'c': row[2], 'd': row[3]})
print datapaths, "process end"
class RedisManager:
def __init__(self, redis_server, pw=None):
#self.redis_client_ = redis.Redis(host='127.0.0.1', port=9966)
if pw:
pool = ClusterConnectionPool(startup_nodes=redis_server, password=pw, skip_full_coverage_check=True, decode_responses=True)
self.redis_client_ = StrictRedisCluster(connection_pool=pool)
else:
pool = ClusterConnectionPool(startup_nodes=redis_server, skip_full_coverage_check=True, decode_responses=True)
self.redis_client_ = StrictRedisCluster(connection_pool=pool)
def Set(self, k, v, is_str, expire=None):
if not is_str:
v = json.dumps(v)
return self.redis_client_.set(k, v, ex=expire)
def Get(self, k):
return self.redis_client_.get(k)
def Delete(self, k):
return self.redis_client_.delete(k)
def HashMultiSet(self, k, d, expire=None):
self.redis_client_.hmset(k, d)
if expire:
self.redis_client_.expire(k, expire)
def HashGetAll(self, k):
return self.redis_client_.hgetall(k)
def Pushback(self, k, v, expire=None):
self.redis_client_.rpush(k, v)
if expire:
self.redis_client_.expire(k, expire)
def SetList(self, k, l, expire=None):
self.redis_client_.rpush(k, *l)
if expire:
self.redis_client_.expire(k, expire)
def SetSet(self, k, v, expire=None):
self.redis_client_.sadd(k, v)
if expire:
self.redis_client_.expire(k, expire)
def SortSetSet(self, k, v, expire=None):
self.redis_client_.zadd(k, v[0], v[1])
if expire:
self.redis_client_.expire(k, expire)
def Handle(self):
return self.redis_client_
import cPickle
import time
from rediscluster import StrictRedisCluster
startup_nodes = [{"host": "127.0.0.1", "port": "7000"}]
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=False)
f = open("/home/smacar-user/Downloads/db.pckl","rb")
mydict = cPickle.load(f)
f.close()
home=time.time()
for key in mydict:
field_list=list(range(0,len(mydict[key])))
val_list=mydict[key].tolist()
data=dict(zip(field_list,val_list))
rc.hmset(str(key),data)
print("end time:",time.time()-home)
break
class RedisCluster:
def __init__(self):
try:
self.rc = StrictRedisCluster(startup_nodes=StartupNodesServer,
decode_responses=True)
except:
traceback.print_exc()
def count_keys(self): # 查询当前库里有多少key
return self.rc.dbsize()
def exists_key(self, key):
return self.rc.exists(key)
def delete_key(self, key):
self.rc.delete(key)
def rename_key(self, key1, key2):
self.rc.rename(key1, key2)
# String操作
def set_key_value(self, key, value):
self.rc.set(key, value)
def get_key_value(self, key): # 没有对应key返回None
return self.rc.get(key)
# Hash操作
def set_hash(self, key, mapping): # mapping为字典, 已存在key会覆盖mapping
self.rc.hmset(key, mapping)
def delete_hash_field(self, key, field): # 删除hash表中某个字段，无论字段是否存在
self.rc.hdel(key, field)
def exists_hash_field(self, key, field): # 检查hash表中某个字段存在
return self.rc.hexists(key, field)
def get_hash_field(self, key, field): # 获取hash表中指定字段的值, 没有返回None
return self.rc.hget(key, field)
def get_hash_all_field(self, key): # 获取hash表中指定key所有字段和值,以字典形式，没有key返回空字典
return self.rc.hgetall(key)
def increase_hash_field(self, key, field,
increment): # 为hash表key某个字段的整数型值增加increment
self.rc.hincrby(key, field, increment)
# List操作
def rpush_into_lst(self, key, value): # url从头至尾入列
self.rc.rpush(key, value)
def lpush_into_lst(self, key, value): # url从尾至头入列
self.rc.lpush(key, value)
def lpop_lst_item(self, key): # 从头取出列表第一个元素，没有返回None
return self.rc.lpop(key)
def blpop_lst_item(
self, key): # 从头取出列表第一个元素(元组形式,值为元祖[1], 元祖[0]为key名)，并设置超时，超时返回None
return self.rc.blpop(key, timeout=1)
def rpop_lst_item(self, key): # 从尾取出列表最后一个元素，没有返回None
return self.rc.rpop(key)
def brpop_lst_item(
self,
key): # 从尾取出列表最后一个元素(元组形式,值为元祖[1], 元祖[0]为key名)，并设置超时，超时返回None
return self.rc.brpop(key, timeout=1)
# Set操作
def add_set(self, key, value):
self.rc.sadd(key, value)
def is_member(self, key, value):
return self.rc.sismember(key, value)
def pop_member(self, key): # 随机移除一个值并返回该值,没有返回None
return self.rc.spop(key)
def pop_members(self, key, num): # 随机取出num个值（非移除），列表形式返回这些值，没有返回空列表
return self.rc.srandmember(key, num)
def remove_member(self, key, value): # 移除集合中指定元素
self.rc.srem(key, value)
def get_all_members(self, key): # 返回集合中全部元素,不删除
return self.rc.smembers(key)
def remove_into(self, key1, key2, value): # 把集合key1中value元素移入集合key2中
self.rc.smove(key1, key2, value)
def count_members(self, key): # 计算集合中成员数量
return self.rc.scard(key)
