class Gtoken():
count = 0
ssdb = SSDB(host='localhost', port=8888)
while True:
post_id = ssdb.get('raw' + str(count))
dapet_komen(post_id)
count += 1
def processList(arg):
keyList = arg[0]
redisHost = arg[1]
redisPort = arg[2]
ssdbHost = arg[3]
ssdbPort = arg[4]
if len(keyList) == 0:
return
try:
#get data from redis
data = {}
r = redis.Redis(host=redisHost, port=redisPort)
pipeRedis = r.pipeline()
#get keys
for item in keyList:
pipeRedis.lrange(item, 0, -1)
retList = pipeRedis.execute()
for i in range(len(retList)):
if len(retList[i]) > 0:
data[keyList[i]] = retList[i]
#put keys
print "get redis data size: %d" % (len(data))
#print data
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.qclear(key)
s.qpush_back(key, *data[key])
print "lpush key: %s" % (key)
#set expire
setExpire(keyList, "list", EX_TIME)
except Exception, e:
print e.message
def processString(arg):
keyList = arg[0]
redisHost = arg[1]
redisPort = arg[2]
ssdbHost = arg[3]
ssdbPort = arg[4]
if len(keyList) == 0:
return
try:
#get data from redis
data = {}
r = redis.Redis(host=redisHost, port=redisPort)
#get keys
retList = r.mget(keyList)
for i in range(len(retList)):
if retList[i] != "(nil)":
data[keyList[i]] = retList[i]
#put keys
print "get redis data size: %d" % (len(data))
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.setx(key, data[key], EX_TIME)
print "set key: %s" % (key)
except Exception, e:
print e.message
def quary():
keyword = "weibo_拾笔不良_2017-05-08"
try:
client = SSDB(host='', port=8884) #连接到服务器的SSDB数据库
res = client.get(keyword)
except Exception as e:
print e.message
results = []
else:
results = json.loads(res)
finally:
print len(results)
for block in results:
print "++++++++++++++"
print "name:", block['name']
" *** 重要 *** "
print "content:", urllib.unquote(str(
block['content'])) #首先将结果转换为字符串，然后对字符串进行url解码
print "point:", block['point']
comment_list = block['comment_list']
for comment in comment_list:
print "comment:", comment
def ssdb_info(self):
try:
ssdb_cli = SSDB(host=self.addr, port=self.port)
ssdb_info_list = ssdb_cli.execute_command("info")[2:-2]
ssdb_info = self.process_ssdb_info_simple(ssdb_info_list)
except Exception as e:
self.logger.error(e)
ssdb_info = {}
return ssdb_info
def __init__(self, name, host, port):
"""
init
:param name: hash name
:param host: ssdb host
:param port: ssdb port
:return:
"""
self.name = name
self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=host, port=port))
def __init__(self, queue_name, host, port, max_connections=10, timeout=60):
self.queue_name = queue_name
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
def __init__(self, id):
self.check_param()
self.mapId_list = []
self.setId_list = []
for param in FLUSH_MAP_ID:
self.mapId_list.append(param + id)
for param in FLUSH_SET_ID:
self.setId_list.append(param + id)
self.ssdb = SSDB(host=SOURCE_IP, port=SOURCE_PORT)
def getIP():
client = SSDB(host='10.141.5.89', port=8884)
keyword = "proxy"
res = client.get(keyword)
addr_list = list()
ip_port_list = res.split(",")[0:-2]
"""拆分出IP和Port"""
for ip_port in ip_port_list:
ip, port = re.split("\s+", ip_port)
addr_list.append(ip + ":" + port)
print "addr parse successful"
return addr_list
def __init__(self, set_name, host, port, max_connections=2, timeout=60):
"""数据库初始化"""
self.set_name = set_name
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
pass
def process(keyword):
addr_list = tools.getIP()
addr = addr_list[random.randint(0, len(addr_list)-1)] # 随机选择一个代理IP
"""代理设置"""
proxy = Proxy(
{
'proxy_type': ProxyType.MANUAL,
'http_proxy': addr
}
)
desired_capabilities = webdriver.DesiredCapabilities.PHANTOMJS
proxy.add_to_capabilities(desired_capabilities)
"""1) 构造driver对象，并设置窗口尺寸"""
driver = webdriver.PhantomJS(desired_capabilities=desired_capabilities)
driver.maximize_window()
"""2) 通过driver对象，实例化Sina类"""
sina = Sina(DRIVER=driver)
"""3) 实现登录，两种登录方式二选一"""
sina.login2()
# sina.login()
"""4) 搜索"""
sina.search(keyword)
"""连接到SSDB数据库"""
client = SSDB(host='', port=8884)
"""获取结果集"""
results = list()
page = 1
while len(results) < 100: #至少返回100条数据
print "+++++++++++++++++++++++++++++++++++++++", "page", page
block_list = sina.extract()
results.extend(block_list)
page += 1
if sina.nextPage():
continue
else:
break
key = "weibo_"+keyword+"_"+tools.getTimeAsStr()
value = json.dumps(results)
client.set(key, value)
print "ssdb save", key, len(results)
def __init__(self,
host="127.0.0.1",
port=8888,
max_connections=10,
timeout=60):
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
def setExpire(keyList, redisType, extime):
if redisType not in ["string", "list", "set", "hash", "sortedset"
] or len(keyList) == 0:
print "key type error! type: %s" % (redisType)
return
data = {}
data[EX_KEY] = {}
for key in keyList:
stringKey = "%s_%s" % (redisType, key)
data[EX_KEY][stringKey] = int(time.time()) + extime
print "expire key: %s" % (stringKey)
print "get expire key size: %d" % (len(data))
#print data
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.multi_zset(key, **data[key])
print "zadd key: %s" % (key)
print "set expire key finish"
def processSet(arg):
keyList = arg[0]
redisHost = arg[1]
redisPort = arg[2]
ssdbHost = arg[3]
ssdbPort = arg[4]
if len(keyList) == 0:
return
try:
#get data from redis
data = {}
r = redis.Redis(host=redisHost, port=redisPort)
pipeRedis = r.pipeline()
#get keys
for item in keyList:
pipeRedis.smembers(item)
retList = pipeRedis.execute()
#print retList
for i in range(len(retList)):
if len(retList[i]) > 0:
data[keyList[i]] = {}
for item in retList[i]:
data[keyList[i]][item] = 0
#put keys
print "get redis data size: %d" % (len(data))
#print data
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.zclear(key)
s.multi_zset(key, **data[key])
print "zadd key: %s" % (key)
#set expire
setExpire(keyList, "sortedset", EX_TIME)
except Exception, e:
print e.message
def ping_ssdb(self):
ssdb_is_alive = 0
try:
ssdb_cli = SSDB(host=self.addr, port=self.port)
ping_res = ssdb_cli.execute_command("ping")
if ping_res:
ssdb_is_alive = 1
except Exception as e:
self.logger.error(e)
if ssdb_is_alive == 0: # If ssdb is dead, update the alive metrice here.
ssdb_alive_data = [{
"endpoint": self.addr,
"metric": "ssdb_alive",
"tags": self.tags,
"timestamp": upload_ts,
"value": ssdb_is_alive,
"step": 60,
"counterType": "GAUGE"
}]
r = requests.post(falcon_client, data=json.dumps(ssdb_alive_data))
self.logger.debug(r.text)
return ssdb_is_alive
#!/usr/bin/env python # -*- coding: utf-8 -*- """ @license : (C) Copyright 2013-2017, Easy doesnt enter into grown-up life. @Software: PyCharm @Project : appstore @Time : 2018/4/5 下午5:18 @Author : achilles_xushy @contact : [email protected] @Site : @File : db.py @desc : """ from ssdb import SSDB # 主业务 db = SSDB(host='127.0.0.1', port=8887, socket_timeout=600)
def __init__(self, host='localhost', port=8888, **kwargs):
QueueBase.QueueBase.__init__(self, host, port)
self.__conn = SSDB(connection_pool=BlockingConnectionPool(
host=self.host, port=self.port))
self.name = kwargs.get('keyName', 'default')
#encoding=utf8
import time
import redis
from ssdb import SSDB
rclient = redis.Redis(host="*", port=36379, password='******')
rpipeline = rclient.pipeline()
pipe_time = time.time()
for x in range(10000):
rpipeline.lpop('task_list_test01')
#rpipeline.lrange('task_list_test01',0,10000)
records = rpipeline.execute()
print('redis total time:', time.time() - pipe_time)
sclient = SSDB('*', port=46379)
# sclient = SSDB(host='18.196.*.*',port=46379,)
sclient.execute_command('auth xxxxxxx')
i = 1
begin_time = time.time()
sclient.qpush_front('task_list_sorted', records)
print(time.time() - begin_time)
exit(0)
for record in records:
# sclient.zset('task_list_sorted',record,i)
sclient.qpush_front('task_list_sorted', record)
i += 1
print(time.time() - begin_time)
#------------------------------------
#postgresql连接代码
def __init__(self, host, port, table_name, **args):
self.queue_name = table_name
pool = BlockingConnectionPool(host=host, port=port, **args)
self.ssdb = SSDB(connection_pool=pool)
print 'success init ssdb connection'
# -*- coding: utf-8 -*-
from ssdb import SSDB
if __name__ == '__main__':
r = SSDB(host='10.68.120.175',
port=7777,
password='******')
# r = SSDB.from_url(url='ssdb://:[email protected]:7777')
# **Server**
# b = r.auth('11111111111111111111111111111111')
# b=r.info('cmd')
# b= r.dbsize()
# b = r.flushdb()
# **Key Value**
# b=r.set('aa',11) # 1
# b=r.setx('aa',11,1000) # 1
# b = r.setnx('bds', 11) # 1/0
# b = r.expire('aaqe',1000) # 1/0
# b = r.ttl('bdasd') # int value
b = r.get('bdas') # value/none
# b = r.getset('bdadssf','22') # value/none
# b = r.delete('aa') # 1
# b = r.incr('aaaasdfsd',2) # int value
# b = r.exists('aaa') # true/false
# b = r.getbit('14aaasdf',11) # 1/0
# b = r.setbit('asfga',11112341, 1) # 1/0
# b = r.bitcount('aavba', 0,1) # 1/0
# b = r.countbit('aa',0,-1) # 1/0
# b = r.substr('bdadssf',0,-1) # value
# b = r.strlen('asfga') # int value
from ssdb import SSDB
import sys
#import cugbacm.proto.rank_pb2
ssdb_ip = "127.0.0.1"
ssdb_port = 6666
ssdb = SSDB(host=ssdb_ip, port=ssdb_port)
def GetContestRankListProto(contestID):
global ssdb
proto_str = ssdb.get(contestID)
return proto_str
def SetContestRankListProto(contestID, rank_list_proto_str):
global ssdb
try:
ssdb.set(contestID, rank_list_proto_str)
except:
pass
def InsertUserProblemStatus(userID, problemID, status):
#AC == 1 No_pass = 2 other = 0
global ssdb
value = "2"
if status == "Accepted":
value = "1"
st = str(ssdb.get(userID + '\t' + str(problemID)))
if st == "1":
def __init__(self, name, host='localhost', port=8888, **kwargs):
QueueBase.QueueBase.__init__(self, name, host, port)
self.__conn = SSDB(connection_pool=BlockingConnectionPool(
host=self.host, port=self.port))
import grequests
from ssdb import SSDB
ssdb = SSDB(host='localhost', port=8888)
def init(howLong):
a = []
for i in range(howLong):
a.append("http://zq.win007.com/jsData/teamInfo/teamDetail/tdl" +
str(i) + ".js")
return a
def downloadJs(url_list):
url = url_list
rs = (grequests.get(u) for u in url)
a = grequests.map(rs)
return a
def insertQueue(q_name, content):
ssdb.qpush_front(q_name, content)
#ssdb.qclear('q')
#i = downloadJs(init(10000))
#for a in i:
# a = a.content
# if a[3:6] == "var":
# insertQueue('q', a)
print ssdb.qpop('q')
