class Gtoken(): count = 0 ssdb = SSDB(host='localhost', port=8888) while True: post_id = ssdb.get('raw' + str(count)) dapet_komen(post_id) count += 1
def processList(arg): keyList = arg[0] redisHost = arg[1] redisPort = arg[2] ssdbHost = arg[3] ssdbPort = arg[4] if len(keyList) == 0: return try: #get data from redis data = {} r = redis.Redis(host=redisHost, port=redisPort) pipeRedis = r.pipeline() #get keys for item in keyList: pipeRedis.lrange(item, 0, -1) retList = pipeRedis.execute() for i in range(len(retList)): if len(retList[i]) > 0: data[keyList[i]] = retList[i] #put keys print "get redis data size: %d" % (len(data)) #print data #put data to ssdb s = SSDB(SSDB_HOST, SSDB_PORT) for key in data: s.qclear(key) s.qpush_back(key, *data[key]) print "lpush key: %s" % (key) #set expire setExpire(keyList, "list", EX_TIME) except Exception, e: print e.message
def processString(arg): keyList = arg[0] redisHost = arg[1] redisPort = arg[2] ssdbHost = arg[3] ssdbPort = arg[4] if len(keyList) == 0: return try: #get data from redis data = {} r = redis.Redis(host=redisHost, port=redisPort) #get keys retList = r.mget(keyList) for i in range(len(retList)): if retList[i] != "(nil)": data[keyList[i]] = retList[i] #put keys print "get redis data size: %d" % (len(data)) #put data to ssdb s = SSDB(SSDB_HOST, SSDB_PORT) for key in data: s.setx(key, data[key], EX_TIME) print "set key: %s" % (key) except Exception, e: print e.message
def quary(): keyword = "weibo_拾笔不良_2017-05-08" try: client = SSDB(host='', port=8884) #连接到服务器的SSDB数据库 res = client.get(keyword) except Exception as e: print e.message results = [] else: results = json.loads(res) finally: print len(results) for block in results: print "++++++++++++++" print "name:", block['name'] " *** 重要 *** " print "content:", urllib.unquote(str( block['content'])) #首先将结果转换为字符串,然后对字符串进行url解码 print "point:", block['point'] comment_list = block['comment_list'] for comment in comment_list: print "comment:", comment
def ssdb_info(self): try: ssdb_cli = SSDB(host=self.addr, port=self.port) ssdb_info_list = ssdb_cli.execute_command("info")[2:-2] ssdb_info = self.process_ssdb_info_simple(ssdb_info_list) except Exception as e: self.logger.error(e) ssdb_info = {} return ssdb_info
def __init__(self, name, host, port): """ init :param name: hash name :param host: ssdb host :param port: ssdb port :return: """ self.name = name self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=host, port=port))
def __init__(self, queue_name, host, port, max_connections=10, timeout=60): self.queue_name = queue_name self.host = host self.port = port self.max_connections = max_connections self.timeout = timeout pool = BlockingConnectionPool(connection_class=Connection, max_connections=max_connections, timeout=timeout, host=host, port=port) self.ssdb = SSDB(connection_pool=pool)
def __init__(self, id): self.check_param() self.mapId_list = [] self.setId_list = [] for param in FLUSH_MAP_ID: self.mapId_list.append(param + id) for param in FLUSH_SET_ID: self.setId_list.append(param + id) self.ssdb = SSDB(host=SOURCE_IP, port=SOURCE_PORT)
def getIP(): client = SSDB(host='10.141.5.89', port=8884) keyword = "proxy" res = client.get(keyword) addr_list = list() ip_port_list = res.split(",")[0:-2] """拆分出IP和Port""" for ip_port in ip_port_list: ip, port = re.split("\s+", ip_port) addr_list.append(ip + ":" + port) print "addr parse successful" return addr_list
def __init__(self, set_name, host, port, max_connections=2, timeout=60): """数据库初始化""" self.set_name = set_name self.host = host self.port = port self.max_connections = max_connections self.timeout = timeout pool = BlockingConnectionPool(connection_class=Connection, max_connections=max_connections, timeout=timeout, host=host, port=port) self.ssdb = SSDB(connection_pool=pool) pass
def process(keyword): addr_list = tools.getIP() addr = addr_list[random.randint(0, len(addr_list)-1)] # 随机选择一个代理IP """代理设置""" proxy = Proxy( { 'proxy_type': ProxyType.MANUAL, 'http_proxy': addr } ) desired_capabilities = webdriver.DesiredCapabilities.PHANTOMJS proxy.add_to_capabilities(desired_capabilities) """1) 构造driver对象,并设置窗口尺寸""" driver = webdriver.PhantomJS(desired_capabilities=desired_capabilities) driver.maximize_window() """2) 通过driver对象,实例化Sina类""" sina = Sina(DRIVER=driver) """3) 实现登录,两种登录方式二选一""" sina.login2() # sina.login() """4) 搜索""" sina.search(keyword) """连接到SSDB数据库""" client = SSDB(host='', port=8884) """获取结果集""" results = list() page = 1 while len(results) < 100: #至少返回100条数据 print "+++++++++++++++++++++++++++++++++++++++", "page", page block_list = sina.extract() results.extend(block_list) page += 1 if sina.nextPage(): continue else: break key = "weibo_"+keyword+"_"+tools.getTimeAsStr() value = json.dumps(results) client.set(key, value) print "ssdb save", key, len(results)
def __init__(self, host="127.0.0.1", port=8888, max_connections=10, timeout=60): self.host = host self.port = port self.max_connections = max_connections self.timeout = timeout pool = BlockingConnectionPool(connection_class=Connection, max_connections=max_connections, timeout=timeout, host=host, port=port) self.ssdb = SSDB(connection_pool=pool)
def setExpire(keyList, redisType, extime): if redisType not in ["string", "list", "set", "hash", "sortedset" ] or len(keyList) == 0: print "key type error! type: %s" % (redisType) return data = {} data[EX_KEY] = {} for key in keyList: stringKey = "%s_%s" % (redisType, key) data[EX_KEY][stringKey] = int(time.time()) + extime print "expire key: %s" % (stringKey) print "get expire key size: %d" % (len(data)) #print data #put data to ssdb s = SSDB(SSDB_HOST, SSDB_PORT) for key in data: s.multi_zset(key, **data[key]) print "zadd key: %s" % (key) print "set expire key finish"
def processSet(arg): keyList = arg[0] redisHost = arg[1] redisPort = arg[2] ssdbHost = arg[3] ssdbPort = arg[4] if len(keyList) == 0: return try: #get data from redis data = {} r = redis.Redis(host=redisHost, port=redisPort) pipeRedis = r.pipeline() #get keys for item in keyList: pipeRedis.smembers(item) retList = pipeRedis.execute() #print retList for i in range(len(retList)): if len(retList[i]) > 0: data[keyList[i]] = {} for item in retList[i]: data[keyList[i]][item] = 0 #put keys print "get redis data size: %d" % (len(data)) #print data #put data to ssdb s = SSDB(SSDB_HOST, SSDB_PORT) for key in data: s.zclear(key) s.multi_zset(key, **data[key]) print "zadd key: %s" % (key) #set expire setExpire(keyList, "sortedset", EX_TIME) except Exception, e: print e.message
def ping_ssdb(self): ssdb_is_alive = 0 try: ssdb_cli = SSDB(host=self.addr, port=self.port) ping_res = ssdb_cli.execute_command("ping") if ping_res: ssdb_is_alive = 1 except Exception as e: self.logger.error(e) if ssdb_is_alive == 0: # If ssdb is dead, update the alive metrice here. ssdb_alive_data = [{ "endpoint": self.addr, "metric": "ssdb_alive", "tags": self.tags, "timestamp": upload_ts, "value": ssdb_is_alive, "step": 60, "counterType": "GAUGE" }] r = requests.post(falcon_client, data=json.dumps(ssdb_alive_data)) self.logger.debug(r.text) return ssdb_is_alive
#!/usr/bin/env python # -*- coding: utf-8 -*- """ @license : (C) Copyright 2013-2017, Easy doesnt enter into grown-up life. @Software: PyCharm @Project : appstore @Time : 2018/4/5 下午5:18 @Author : achilles_xushy @contact : [email protected] @Site : @File : db.py @desc : """ from ssdb import SSDB # 主业务 db = SSDB(host='127.0.0.1', port=8887, socket_timeout=600)
def __init__(self, host='localhost', port=8888, **kwargs): QueueBase.QueueBase.__init__(self, host, port) self.__conn = SSDB(connection_pool=BlockingConnectionPool( host=self.host, port=self.port)) self.name = kwargs.get('keyName', 'default')
#encoding=utf8 import time import redis from ssdb import SSDB rclient = redis.Redis(host="*", port=36379, password='******') rpipeline = rclient.pipeline() pipe_time = time.time() for x in range(10000): rpipeline.lpop('task_list_test01') #rpipeline.lrange('task_list_test01',0,10000) records = rpipeline.execute() print('redis total time:', time.time() - pipe_time) sclient = SSDB('*', port=46379) # sclient = SSDB(host='18.196.*.*',port=46379,) sclient.execute_command('auth xxxxxxx') i = 1 begin_time = time.time() sclient.qpush_front('task_list_sorted', records) print(time.time() - begin_time) exit(0) for record in records: # sclient.zset('task_list_sorted',record,i) sclient.qpush_front('task_list_sorted', record) i += 1 print(time.time() - begin_time) #------------------------------------ #postgresql连接代码
def __init__(self, host, port, table_name, **args): self.queue_name = table_name pool = BlockingConnectionPool(host=host, port=port, **args) self.ssdb = SSDB(connection_pool=pool) print 'success init ssdb connection'
# -*- coding: utf-8 -*- from ssdb import SSDB if __name__ == '__main__': r = SSDB(host='10.68.120.175', port=7777, password='******') # r = SSDB.from_url(url='ssdb://:[email protected]:7777') # **Server** # b = r.auth('11111111111111111111111111111111') # b=r.info('cmd') # b= r.dbsize() # b = r.flushdb() # **Key Value** # b=r.set('aa',11) # 1 # b=r.setx('aa',11,1000) # 1 # b = r.setnx('bds', 11) # 1/0 # b = r.expire('aaqe',1000) # 1/0 # b = r.ttl('bdasd') # int value b = r.get('bdas') # value/none # b = r.getset('bdadssf','22') # value/none # b = r.delete('aa') # 1 # b = r.incr('aaaasdfsd',2) # int value # b = r.exists('aaa') # true/false # b = r.getbit('14aaasdf',11) # 1/0 # b = r.setbit('asfga',11112341, 1) # 1/0 # b = r.bitcount('aavba', 0,1) # 1/0 # b = r.countbit('aa',0,-1) # 1/0 # b = r.substr('bdadssf',0,-1) # value # b = r.strlen('asfga') # int value
from ssdb import SSDB import sys #import cugbacm.proto.rank_pb2 ssdb_ip = "127.0.0.1" ssdb_port = 6666 ssdb = SSDB(host=ssdb_ip, port=ssdb_port) def GetContestRankListProto(contestID): global ssdb proto_str = ssdb.get(contestID) return proto_str def SetContestRankListProto(contestID, rank_list_proto_str): global ssdb try: ssdb.set(contestID, rank_list_proto_str) except: pass def InsertUserProblemStatus(userID, problemID, status): #AC == 1 No_pass = 2 other = 0 global ssdb value = "2" if status == "Accepted": value = "1" st = str(ssdb.get(userID + '\t' + str(problemID))) if st == "1":
def __init__(self, name, host='localhost', port=8888, **kwargs): QueueBase.QueueBase.__init__(self, name, host, port) self.__conn = SSDB(connection_pool=BlockingConnectionPool( host=self.host, port=self.port))
import grequests from ssdb import SSDB ssdb = SSDB(host='localhost', port=8888) def init(howLong): a = [] for i in range(howLong): a.append("http://zq.win007.com/jsData/teamInfo/teamDetail/tdl" + str(i) + ".js") return a def downloadJs(url_list): url = url_list rs = (grequests.get(u) for u in url) a = grequests.map(rs) return a def insertQueue(q_name, content): ssdb.qpush_front(q_name, content) #ssdb.qclear('q') #i = downloadJs(init(10000)) #for a in i: # a = a.content # if a[3:6] == "var": # insertQueue('q', a) print ssdb.qpop('q')