def processList(arg):
keyList = arg[0]
redisHost = arg[1]
redisPort = arg[2]
ssdbHost = arg[3]
ssdbPort = arg[4]
if len(keyList) == 0:
return
try:
#get data from redis
data = {}
r = redis.Redis(host=redisHost, port=redisPort)
pipeRedis = r.pipeline()
#get keys
for item in keyList:
pipeRedis.lrange(item, 0, -1)
retList = pipeRedis.execute()
for i in range(len(retList)):
if len(retList[i]) > 0:
data[keyList[i]] = retList[i]
#put keys
print "get redis data size: %d" % (len(data))
#print data
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.qclear(key)
s.qpush_back(key, *data[key])
print "lpush key: %s" % (key)
#set expire
setExpire(keyList, "list", EX_TIME)
except Exception, e:
print e.message
class myssdb(object):
def __init__(self, table=None, host='localhost', port=8888, **kwargs):
self.table = table
pool = BlockingConnectionPool(host=host, port=port, **kwargs)
self.ssdb = SSDB(connection_pool=pool)
# 队尾放入一条数据
def put(self, data, **kwargs):
length = 0
if isinstance(data, dict) or isinstance(data, list):
length = self.ssdb.qpush_back(self.table, json.dumps(data))
else:
length = self.ssdb.qpush_back(self.table, data)
if length:
self.ssdb.hincr('counter_' + self.table, time.strftime('%Y-%m-%d', time.localtime(float(time.time()))))
return length
# 队首取出一条数据
def get(self, **kwargs):
data = self.ssdb.qpop_front(self.table)
if data:
return data[0]
return data
class QueueSSDB(QueueBase.QueueBase):
def __init__(self, name, host='localhost', port=8888, **kwargs):
QueueBase.QueueBase.__init__(self, name, host, port)
self.__conn = SSDB(connection_pool=BlockingConnectionPool(
host=self.host, port=self.port))
@QueueBase.catch
def put(self, value, *args, **kwargs):
return self.__conn.qpush_back(
self.name,
json.dumps(value)
if isinstance(value, dict) or isinstance(value, list) else value)
@QueueBase.catch
def get(self, *args, **kwargs):
value = self.__conn.qpop_front(self.name)
return value[0] if value else value
@QueueBase.catch
def size(self, *args, **kwargs):
return self.__conn.qsize(self.name)
class ssdb(storage):
def __init__(self, host, port, table_name, **args):
self.queue_name = table_name
pool = BlockingConnectionPool(host=host, port=port, **args)
self.ssdb = SSDB(connection_pool=pool)
print 'success init ssdb connection'
def save(self, data, **args):
"""
push data to ssdb queue back
:param data: data of dict type
:return:（None)
"""
while True:
try:
if isinstance(data, dict) or isinstance(data, list):
self.ssdb.qpush_back(self.queue_name, json.dumps(data))
else:
self.ssdb.qpush_back(self.queue_name, data)
break
except Exception as e:
print e
time.sleep(60)
def get(self, **args):
while True:
try:
data = self.ssdb.qpop_front(self.queue_name)
if data:
return data[0]
return data
except Exception as e:
print e
time.sleep(60)
# 切换队列
def select_queue(self, name):
self.queue_name = name
def size(self, **args):
return self.ssdb.qsize(self.queue_name)
def len(self, **args):
return self.ssdb.qsize(self.queue_name)
def put(self, data, **args):
"""
push data to ssdb queue back
:param data: data of dict type
:return:（None)
"""
while True:
try:
if isinstance(data, dict) or isinstance(data, list):
self.ssdb.qpush_back(self.queue_name, json.dumps(data))
else:
self.ssdb.qpush_back(self.queue_name, data)
break
except Exception as e:
print e
time.sleep(60)
def put_data_back(self, data):
return self.ssdb.qpush_back(self.queue_name, data)
def put_data_front(self, data):
return self.ssdb.qpush_front(self.queue_name, data)
def ssdb_put_zset(self, field):
#zset zset key score
while True:
try:
return self.ssdb.zset(self.queue_name, field)
except Exception as e:
print e
self.SSDB()
def ssdb_zexists(self, field):
while True:
try:
return self.ssdb.zexists(self.queue_name, field)
except Exception as e:
print e
self.SSDB()
def ssdb_zdel(self, field):
while True:
try:
return self.ssdb.zdel(self.queue_name, field)
except Exception as e:
print e
self.SSDB()
class SSDBQueue(object):
def __init__(self, queue_name, host, port, max_connections=10, timeout=60):
self.queue_name = queue_name
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
def queue_size(self):
return self.ssdb.qsize(self.queue_name)
def clear_queue(self):
return self.ssdb.qclear(self.queue_name)
def empty(self):
return self.ssdb.qsize(self.queue_name) == 0
def put_data(self, data):
return self.ssdb.qpush_back(self.queue_name, data)
def get_data(self):
return self.ssdb.qpop_front(self.queue_name)
def put(self,
data,
sleep_time=10,
retry=9,
email_list=functions.mailto_list_ourselves):
i = 0
count = retry + 1
ssdb_queue = None
while True:
try:
if data != None and ssdb_queue != None:
if isinstance(data, dict):
return ssdb_queue.put_data(pickle.dumps(data))
else:
if isinstance(data, dict):
return self.put_data(pickle.dumps(data))
except Exception as e:
print u'插入队列异常 %s' % exceputil.traceinfo(e)
i += 1
if i >= count:
#发邮件
functions.send_mail_old(
email_list, u"ssdb队列更新异常", u"错误信息%s \nqueue_name:%s" %
(exceputil.traceinfo(e), self.queue_name))
time.sleep(600)
i = 0
else:
time.sleep(sleep_time)
ssdb_queue = getSSDBQueuev2(self.queue_name, self.host,
self.port, self.max_connections,
self.timeout)
def get(self,
sleep_time=10,
retry=9,
email_list=functions.mailto_list_ourselves):
i = 0
count = retry + 1
ssdb_queue = None
while True:
try:
if ssdb_queue != None:
return ssdb_queue.get_data()
else:
return self.get_data()
except Exception as e:
print u'查询数据异常 %s' % exceputil.traceinfo(e)
i += 1
if i >= count:
#发邮件
functions.send_mail_old(email_list, u"ssdb队列更新异常",
u"错误信息%s" % exceputil.traceinfo(e))
time.sleep(600)
i = 0
else:
time.sleep(sleep_time)
ssdb_queue = getSSDBQueuev2(self.queue_name, self.host,
self.port, self.max_connections,
self.timeout)
def size(self,
sleep_time=10,
retry=9,
email_list=functions.mailto_list_ourselves):
i = 0
count = retry + 1
ssdb_queue = None
while True:
try:
if ssdb_queue != None:
return ssdb_queue.queue_size()
else:
return self.queue_size()
except Exception as e:
print u'查询队列中数异常 %s' % exceputil.traceinfo(e)
i += 1
if i >= count:
#发邮件
functions.send_mail_old(email_list, u"ssdb队列更新异常",
u"错误信息%s" % exceputil.traceinfo(e))
time.sleep(600)
i = 0
else:
time.sleep(sleep_time)
ssdb_queue = getSSDBQueuev2(self.queue_name, self.host,
self.port, self.max_connections,
self.timeout)
def clear(self,
sleep_time=10,
retry=9,
email_list=functions.mailto_list_ourselves):
i = 0
count = retry + 1
ssdb_queue = None
while True:
try:
if ssdb_queue != None:
return ssdb_queue.clear_queue()
else:
return self.clear_queue()
except Exception as e:
print u'查询队列中数异常 %s' % exceputil.traceinfo(e)
i += 1
if i >= count:
#发邮件
functions.send_mail_old(email_list, u"ssdb队列更新异常",
u"错误信息%s" % exceputil.traceinfo(e))
time.sleep(600)
i = 0
else:
time.sleep(sleep_time)
ssdb_queue = getSSDBQueuev2(self.queue_name, self.host,
self.port, self.max_connections,
self.timeout)
class QueueSSDB(QueueBase.QueueBase):
"""
base class , only provide interface for sub class to implement
"""
def __init__(self, name, host='localhost', port=8888, **kwargs):
QueueBase.QueueBase.__init__(self, name, host, port)
self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=self.host, port=self.port))
#queue
@QueueBase.catch
def put(self, value, *args, **kwargs):
"""
put an item in the back of a queue
:param value:
:param args:
:param kwargs:
:return:
"""
return self.__conn.qpush_back(self.name,
json.dumps(value, ensure_ascii=False).encode('utf-8') if isinstance(value, dict) or isinstance(value, list) else value)
def save(self, value, *args, **kwargs):
"""
put an item in the back of a queue
:param value:
:param args:
:param kwargs:
:return:
"""
return self.__conn.qpush_back(self.name,
json.dumps(value, ensure_ascii = False).encode('utf-8') if isinstance(value,
dict) or isinstance(
value, list) else value)
@QueueBase.catch
def get(self, *args, **kwargs):
"""
get element from the from of queue
:param args:
:param kwargs:
:return:
"""
value = self.__conn.qpop_front(self.name)
return value[0] if value else value
@QueueBase.catch
def getMore(self, *args, **kwargs):
"""
get element from the from of queue
:param args:
:param kwargs:
:return:
"""
value = self.__conn.qpop_front(self.name, **kwargs)
return value
@QueueBase.catch
def size(self, *args, **kwargs):
return self.__conn.qsize(self.name)
@QueueBase.catch
def changeTable(self, name):
"""
change the queue name to operate
:param name:
:return:
"""
self.name = name
@QueueBase.catch
def select_queue(self, name):
"""
change the queue name to operate
:param name:
:return:
"""
self.name = name
@QueueBase.catch
def qclaerQueue(self):
return self.__conn.qclear(self.name)
#KV
@QueueBase.catch
def keySet(self,key,value):
"""
Set the value at key ``name`` to ``value`` .
:param key:
:param value:
:return:
"""
value = json.dumps(value, ensure_ascii = False).encode('utf-8') if isinstance(value,dict) or isinstance(value, list) else value
return self.__conn.set(key,value)
@QueueBase.catch
def keySetx(self,name, value, ttl=-1):
"""
Set the value of key ``name`` to ``value`` that expires in ``ttl``
seconds. ``ttl`` can be represented by an integer or a Python
timedelta object.
:param name:
:param value:
:param ttl:
:return:
"""
return self.__conn.setx(name,value,ttl=ttl)
@QueueBase.catch
def keyTtl(self,key):
"""
Returns the number of seconds until the key ``name`` will expire.
:return:
"""
self.__conn.ttl(key)
@QueueBase.catch
def keyGet(self,key):
"""
Return the value at key ``name``, or ``None`` if the key doesn't exist
:param key:
:return:
"""
return self.__conn.get(key)
@QueueBase.catch
def keyDel(self,key):
"""
Delete the key specified by ``name`` .
:param key:
:return:
"""
return self.__conn.delete(key)
@QueueBase.catch
def keyKeys(self,key_start='',key_end=''):
"""
Return a list of the top ``limit`` keys between ``name_start`` and
``name_end``
:param key_start:
:param key_end:
:return:
"""
return self.__conn.keys(name_start=key_start,name_end=key_end,limit=100000)
@QueueBase.catch
def keyexists(self,key):
"""
:param key:
:return:
"""
return self.__conn.exists(key)
#SET
@QueueBase.catch
def zsetSet(self,field,score = 1):
if field:
if isinstance(field, dict) or isinstance(field, list):
field = json.dumps(field)
field = field if len(field) < 100 else field[:100]
return self.__conn.zset(self.name, field, score)
@QueueBase.catch
def zgetSet(self,key):
return self.__conn.zget(self.name,key)
@QueueBase.catch
def zexistsSet(self,name,field):
return self.__conn.zexists(name,field)
@QueueBase.catch
def zkeysSet(self):
return self.__conn.zkeys(self.name,'','','',limit=100000000)
@QueueBase.catch
def zdelSet(self,key):
return self.__conn.zdel(self.name,key)
@QueueBase.catch
def multi_zgetSet(self,*keys):
return self.__conn.multi_zget(self.name,*keys)
#Hash
@QueueBase.catch
def hgetallHash(self,key):
return self.__conn.hgetall(key)
@QueueBase.catch
def hincrHash(self,name,key):
return self.__conn.hincr(name,key,amount=1)
@QueueBase.catch
def multi_hsetHash(self,name,**mapping):
return self.__conn.multi_hset(name, **mapping)
@QueueBase.catch
def hlistHash(self,start,end):
return self.__conn.hlist(start, end, limit = 10000000)
@QueueBase.catch
def hclearHash(self,key):
return self.__conn.hclear(key)
@QueueBase.catch
def hset(self, key,value):
return self.__conn.hset(self.name, key,json.dumps(value, ensure_ascii = False).encode('utf-8')
if isinstance(value,dict) or isinstance(value, list) else value)
@QueueBase.catch
def hsize(self):
return self.__conn.hsize(self.name)
@QueueBase.catch
def hget(self, key = None):
if key:
return self.__conn.hget(self.name, key)
else:
if self.__conn.hsize(self.name) > 0:
keys=self.__conn.hkeys(self.name,"", "",limit=1)
if keys:
key=keys[0]
v=self.__conn.hget(self.name, key)
self.__conn.hdel(self.name, key)
return v