class myssdb(object):
def __init__(self, table=None, host='localhost', port=8888, **kwargs):
self.table = table
pool = BlockingConnectionPool(host=host, port=port, **kwargs)
self.ssdb = SSDB(connection_pool=pool)
# 队尾放入一条数据
def put(self, data, **kwargs):
length = 0
if isinstance(data, dict) or isinstance(data, list):
length = self.ssdb.qpush_back(self.table, json.dumps(data))
else:
length = self.ssdb.qpush_back(self.table, data)
if length:
self.ssdb.hincr('counter_' + self.table, time.strftime('%Y-%m-%d', time.localtime(float(time.time()))))
return length
# 队首取出一条数据
def get(self, **kwargs):
data = self.ssdb.qpop_front(self.table)
if data:
return data[0]
return data
def quary():
keyword = "weibo_拾笔不良_2017-05-08"
try:
client = SSDB(host='', port=8884) #连接到服务器的SSDB数据库
res = client.get(keyword)
except Exception as e:
print e.message
results = []
else:
results = json.loads(res)
finally:
print len(results)
for block in results:
print "++++++++++++++"
print "name:", block['name']
" *** 重要 *** "
print "content:", urllib.unquote(str(
block['content'])) #首先将结果转换为字符串，然后对字符串进行url解码
print "point:", block['point']
comment_list = block['comment_list']
for comment in comment_list:
print "comment:", comment
class SSDBKV(object):
def __init__(self,
host="127.0.0.1",
port=8888,
max_connections=10,
timeout=60):
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
def set(self, key, value):
return self.ssdb.set(key, value)
def get(self, key):
return self.ssdb.get(key)
def delete(self, key):
return self.ssdb.delete(key)
def keys(self, name_start=0, name_end=0xFFFFFFFF, limit=10):
return self.ssdb.keys(name_start, name_end, limit)
def exists(self, key):
return self.ssdb.exists(key)
def processString(arg):
keyList = arg[0]
redisHost = arg[1]
redisPort = arg[2]
ssdbHost = arg[3]
ssdbPort = arg[4]
if len(keyList) == 0:
return
try:
#get data from redis
data = {}
r = redis.Redis(host=redisHost, port=redisPort)
#get keys
retList = r.mget(keyList)
for i in range(len(retList)):
if retList[i] != "(nil)":
data[keyList[i]] = retList[i]
#put keys
print "get redis data size: %d" % (len(data))
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.setx(key, data[key], EX_TIME)
print "set key: %s" % (key)
except Exception, e:
print e.message
def ssdb_info(self):
try:
ssdb_cli = SSDB(host=self.addr, port=self.port)
ssdb_info_list = ssdb_cli.execute_command("info")[2:-2]
ssdb_info = self.process_ssdb_info_simple(ssdb_info_list)
except Exception as e:
self.logger.error(e)
ssdb_info = {}
return ssdb_info
def __init__(self, name, host, port):
"""
init
:param name: hash name
:param host: ssdb host
:param port: ssdb port
:return:
"""
self.name = name
self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=host, port=port))
def __init__(self, queue_name, host, port, max_connections=10, timeout=60):
self.queue_name = queue_name
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
def __init__(self, id):
self.check_param()
self.mapId_list = []
self.setId_list = []
for param in FLUSH_MAP_ID:
self.mapId_list.append(param + id)
for param in FLUSH_SET_ID:
self.setId_list.append(param + id)
self.ssdb = SSDB(host=SOURCE_IP, port=SOURCE_PORT)
def getIP():
client = SSDB(host='10.141.5.89', port=8884)
keyword = "proxy"
res = client.get(keyword)
addr_list = list()
ip_port_list = res.split(",")[0:-2]
"""拆分出IP和Port"""
for ip_port in ip_port_list:
ip, port = re.split("\s+", ip_port)
addr_list.append(ip + ":" + port)
print "addr parse successful"
return addr_list
def __init__(self, set_name, host, port, max_connections=2, timeout=60):
"""数据库初始化"""
self.set_name = set_name
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
pass
def process(keyword):
addr_list = tools.getIP()
addr = addr_list[random.randint(0, len(addr_list)-1)] # 随机选择一个代理IP
"""代理设置"""
proxy = Proxy(
{
'proxy_type': ProxyType.MANUAL,
'http_proxy': addr
}
)
desired_capabilities = webdriver.DesiredCapabilities.PHANTOMJS
proxy.add_to_capabilities(desired_capabilities)
"""1) 构造driver对象，并设置窗口尺寸"""
driver = webdriver.PhantomJS(desired_capabilities=desired_capabilities)
driver.maximize_window()
"""2) 通过driver对象，实例化Sina类"""
sina = Sina(DRIVER=driver)
"""3) 实现登录，两种登录方式二选一"""
sina.login2()
# sina.login()
"""4) 搜索"""
sina.search(keyword)
"""连接到SSDB数据库"""
client = SSDB(host='', port=8884)
"""获取结果集"""
results = list()
page = 1
while len(results) < 100: #至少返回100条数据
print "+++++++++++++++++++++++++++++++++++++++", "page", page
block_list = sina.extract()
results.extend(block_list)
page += 1
if sina.nextPage():
continue
else:
break
key = "weibo_"+keyword+"_"+tools.getTimeAsStr()
value = json.dumps(results)
client.set(key, value)
print "ssdb save", key, len(results)
def __init__(self,
host="127.0.0.1",
port=8888,
max_connections=10,
timeout=60):
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
class Gtoken():
count = 0
ssdb = SSDB(host='localhost', port=8888)
while True:
post_id = ssdb.get('raw' + str(count))
dapet_komen(post_id)
count += 1
class SsdbClient(object):
"""
SSDB client
"""
def __init__(self, name, host, port):
"""
init
:param name:
:param host:
:param port:
:return:
"""
self.name = name
self.__conn = SSDB(
connection_pool=BlockingConnectionPool(host=host, port=port))
def get(self):
"""
get an item
:return:
"""
values = self.__conn.hgetall(name=self.name)
return random.choice(values.keys()) if values else None
def put(self, value):
"""
put an item
:param value:
:return:
"""
value = json.dump(value,
ensure_ascii=False).encode('utf-8') if isinstance(
value, (dict, list)) else value
return self.__conn.hset(self.name, value, None)
def pop(self):
"""
pop an item
:return:
"""
key = self.get()
if key:
self.__conn.hdel(self.name, key)
return key
def delete(self, key):
"""
delete an item
:param key:
:return:
"""
self.__conn.hdel(self.name, key)
def getAll(self):
return self.__conn.hgetall(self.name).keys()
def changeTable(self, name):
self.name = name
def __init__(self, name, host, port):
"""
init
:param name: hash name
:param host: ssdb host
:param port: ssdb port
:return:
"""
self.name = name
self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=host, port=port))
class SSDBHashMap(object, SSDBBase):
def __init__(self,
hashmap_name,
host="127.0.0.1",
port=8888,
max_connections=10,
timeout=60):
self.hashmap_name = hashmap_name
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
SSDBBase.__init__(self, self.ssdb)
def set(self, key, value):
return self.ssdb.hset(self.hashmap_name, key, value)
def get(self, key):
return self.ssdb.hget(self.hashmap_name, key)
def delete(self, key):
return self.ssdb.hdel(self.hashmap_name, key)
def keys(self, name_start="", name_end="", limit=10):
return self.ssdb.hkeys(self.hashmap_name, name_start, name_end, limit)
def exists(self, key):
return self.ssdb.hexists(self.hashmap_name, key)
def size(self):
return self.ssdb.hsize(self.hashmap_name)
def list(self, name_start="", name_end="", limit=10):
#列出名字处于区间 (name_start, name_end] 的 hashmap
return self.ssdb.hlist(name_start, name_end, limit)
def scan(self, key_start, key_end="", limit=10):
return self.ssdb.hscan(self.hashmap_name, key_start, key_end, limit)
def clear(self):
return self.ssdb.hclear(self.hashmap_name)
def setExpire(keyList, redisType, extime):
if redisType not in ["string", "list", "set", "hash", "sortedset"
] or len(keyList) == 0:
print "key type error! type: %s" % (redisType)
return
data = {}
data[EX_KEY] = {}
for key in keyList:
stringKey = "%s_%s" % (redisType, key)
data[EX_KEY][stringKey] = int(time.time()) + extime
print "expire key: %s" % (stringKey)
print "get expire key size: %d" % (len(data))
#print data
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.multi_zset(key, **data[key])
print "zadd key: %s" % (key)
print "set expire key finish"
class myssdb_hash(object):
def __init__(self, table, host="localhost", port=8888, **kwargs):
self.name = table
pool = BlockingConnectionPool(host=host, port=port, **kwargs)
self.__conn = SSDB(connection_pool=pool)
# put a data
def put(self, value, **kwargs):
if isinstance(value, dict):
key = value.get("_id", str(uuid.uuid1()))
elif isinstance(value, basestring):
try:
tmp = json.loads(value)
if isinstance(tmp, dict):
key = tmp.get("_id", str(uuid.uuid1()))
else:
key = str(uuid.uuid1())
except:
key = str(uuid.uuid1())
else:
key = str(uuid.uuid1())
status = self.__conn.hset(self.name, key, value if isinstance(value, basestring) else json.dumps(value))
self.__conn.hincr("counter_" + self.name, time.strftime("%Y-%m-%d", time.localtime(float(time.time()))))
return status
# get a data
def get(self, **kwargs):
tmp = self.__conn.hscan(self.name, "", "", limit=1)
if not tmp:
return None
try:
self.__conn.hdel(self.name, tmp.keys()[0])
except Exception as e:
print str(e)
return None
return tmp.values()[0]
# table size
def size(self, *args, **kwargs):
return self.__conn.hsize(self.name)
def __list(self):
start_key = ""
for i in xrange(0, self.size(), 1000):
tmp = self.__conn.hscan(self.name, start_key, "", limit=1000)
start_key = tmp.keys()[-1]
for value in tmp.values():
yield value
def get_values(self):
return self.__list()
class QueueSSDB(QueueBase.QueueBase):
def __init__(self, name, host='localhost', port=8888, **kwargs):
QueueBase.QueueBase.__init__(self, name, host, port)
self.__conn = SSDB(connection_pool=BlockingConnectionPool(
host=self.host, port=self.port))
@QueueBase.catch
def put(self, value, *args, **kwargs):
return self.__conn.qpush_back(
self.name,
json.dumps(value)
if isinstance(value, dict) or isinstance(value, list) else value)
@QueueBase.catch
def get(self, *args, **kwargs):
value = self.__conn.qpop_front(self.name)
return value[0] if value else value
@QueueBase.catch
def size(self, *args, **kwargs):
return self.__conn.qsize(self.name)
def processList(arg):
keyList = arg[0]
redisHost = arg[1]
redisPort = arg[2]
ssdbHost = arg[3]
ssdbPort = arg[4]
if len(keyList) == 0:
return
try:
#get data from redis
data = {}
r = redis.Redis(host=redisHost, port=redisPort)
pipeRedis = r.pipeline()
#get keys
for item in keyList:
pipeRedis.lrange(item, 0, -1)
retList = pipeRedis.execute()
for i in range(len(retList)):
if len(retList[i]) > 0:
data[keyList[i]] = retList[i]
#put keys
print "get redis data size: %d" % (len(data))
#print data
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.qclear(key)
s.qpush_back(key, *data[key])
print "lpush key: %s" % (key)
#set expire
setExpire(keyList, "list", EX_TIME)
except Exception, e:
print e.message
class QueueHash(QueueBase.QueueBase):
def __init__(self, name, host='localhost', port=8888, **kwargs):
QueueBase.QueueBase.__init__(self, name, host, port)
self.__conn = SSDB(connection_pool=BlockingConnectionPool(
host=self.host, port=self.port))
@QueueBase.catch
def put(self, value, *args, **kwargs):
if isinstance(value, dict):
key = value.get('_id', str(uuid.uuid1()))
elif isinstance(value, basestring):
try:
tmp = json.loads(value)
if isinstance(tmp, dict):
key = tmp.get('_id', str(uuid.uuid1()))
else:
key = str(uuid.uuid1())
except:
key = str(uuid.uuid1())
else:
key = str(uuid.uuid1())
return self.__conn.hset(
self.name, key, value if isinstance(value, basestring) else
json.dumps(value, ensure_ascii=False).encode('utf-8'))
@QueueBase.catch
def get(self, *args, **kwargs):
tmp = self.__conn.hscan(self.name, '', '', limit=1)
if not tmp:
return None
try:
self.__conn.hdel(self.name, tmp.keys()[0])
except Exception as e:
print str(e)
return None
return tmp.values()[0]
@QueueBase.catch
def size(self, *args, **kwargs):
return self.__conn.hsize(self.name)
class Clear(object):
def __init__(self, id):
self.check_param()
self.mapId_list = []
self.setId_list = []
for param in FLUSH_MAP_ID:
self.mapId_list.append(param + id)
for param in FLUSH_SET_ID:
self.setId_list.append(param + id)
self.ssdb = SSDB(host=SOURCE_IP, port=SOURCE_PORT)
def check_param(self):
if CHARACTER_SET == "" or SOURCE_IP == "" or SOURCE_PORT == "":
print("Check Clear Parameters!")
sys.exit()
for param in FLUSH_MAP_ID:
if param == "":
print("Check Clear Parameters!")
for param in FLUSH_SET_ID:
if param == "":
print("Check Clear Parameters!")
def show_config(self):
print("")
print("SOURCE_IP = %s" % SOURCE_IP)
print("SOURCE_PORT = %s" % SOURCE_PORT)
print("")
index = 1
for param in self.mapId_list:
print("mapId%s = %s,Size = %s" %
(index, param, self.ssdb.hsize(param)))
index = index + 1
print("")
index = 1
for param in self.setId_list:
print("setId%s = %s,Size = %s" %
(index, param, self.ssdb.zsize(param)))
index = index + 1
print("")
def clear_data(self):
for param in self.mapId_list:
self.ssdb.hclear(param)
print("Clear %s Success!" % (param))
for param in self.setId_list:
self.ssdb.zclear(param)
print("Clear %s Success!" % (param))
print("Clear SSDB Success!")
def ping_ssdb(self):
ssdb_is_alive = 0
try:
ssdb_cli = SSDB(host=self.addr, port=self.port)
ping_res = ssdb_cli.execute_command("ping")
if ping_res:
ssdb_is_alive = 1
except Exception as e:
self.logger.error(e)
if ssdb_is_alive == 0: # If ssdb is dead, update the alive metrice here.
ssdb_alive_data = [{
"endpoint": self.addr,
"metric": "ssdb_alive",
"tags": self.tags,
"timestamp": upload_ts,
"value": ssdb_is_alive,
"step": 60,
"counterType": "GAUGE"
}]
r = requests.post(falcon_client, data=json.dumps(ssdb_alive_data))
self.logger.debug(r.text)
return ssdb_is_alive
class SSDBZset(object, SSDBBase):
def __init__(self,
zset_name,
host="127.0.0.1",
port=8888,
max_connections=10,
timeout=60):
self.zset_name = zset_name
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
SSDBBase.__init__(self, self.ssdb)
def set(self, key, value, score):
return self.ssdb.zset(self.zset_name, key, score)
def get(self, key):
return self.ssdb.zget(self.zset_name, key)
def delete(self, key):
return self.ssdb.zdel(self.zset_name, key)
def keys(self, key_start="", score_start="", score_end="", limit=10):
return self.ssdb.zkeys(self.zset_name, key_start, score_start,
score_end, limit)
def exists(self, key):
return self.ssdb.zexists(self.zset_name, key)
def size(self):
return self.ssdb.zsize(self.zset_name)
def list(self, name_start="", name_end="", limit=10):
#列出名字处于区间 (name_start, name_end] 的 hashmap
return self.ssdb.zlist(name_start, name_end, limit)
def clear(self):
return self.ssdb.zclear(self.zset_name)
class ssdb_proxy(object):
def __init__(self, set_name, host, port, max_connections=2, timeout=60):
"""数据库初始化"""
self.set_name = set_name
self.host = host
self.port = port
self.max_connections = max_connections
self.timeout = timeout
pool = BlockingConnectionPool(connection_class=Connection,
max_connections=max_connections,
timeout=timeout,
host=host,
port=port)
self.ssdb = SSDB(connection_pool=pool)
pass
def hset(self, key, value):
"""添加集合"""
return self.ssdb.hset(self.set_name, key, value)
def hget(self, key):
"""获取key所对应的值"""
return self.ssdb.hget(self.set_name, key)
def hgetall(self):
"""获取所有的数据"""
return self.ssdb.hgetall(self.set_name)
def hdel(self, key):
"""删除数据"""
return self.ssdb.hdel(self.set_name, key)
def hexists(self, key):
"""判断Key是否存在"""
return self.ssdb.hexists(self.set_name, key)
def size(self):
"""获取结合的大小"""
return self.ssdb.hsize(self.set_name)
def clean(self):
"""清楚所有数据"""
return self.ssdb.hclear(self.set_name)
def processSet(arg):
keyList = arg[0]
redisHost = arg[1]
redisPort = arg[2]
ssdbHost = arg[3]
ssdbPort = arg[4]
if len(keyList) == 0:
return
try:
#get data from redis
data = {}
r = redis.Redis(host=redisHost, port=redisPort)
pipeRedis = r.pipeline()
#get keys
for item in keyList:
pipeRedis.smembers(item)
retList = pipeRedis.execute()
#print retList
for i in range(len(retList)):
if len(retList[i]) > 0:
data[keyList[i]] = {}
for item in retList[i]:
data[keyList[i]][item] = 0
#put keys
print "get redis data size: %d" % (len(data))
#print data
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.zclear(key)
s.multi_zset(key, **data[key])
print "zadd key: %s" % (key)
#set expire
setExpire(keyList, "sortedset", EX_TIME)
except Exception, e:
print e.message
def __init__(self, host='localhost', port=8888, **kwargs):
QueueBase.QueueBase.__init__(self, host, port)
self.__conn = SSDB(connection_pool=BlockingConnectionPool(
host=self.host, port=self.port))
self.name = kwargs.get('keyName', 'default')
import arrow
from ssdb import SSDB
import json
ssdb = SSDB(host='104.131.54.255', port=8888)
onlyfiles = [f for f in listdir(".") if isfile(join(".", f))]
onlyfiles = [i for i in onlyfiles if "results-" in i]
for file_name in onlyfiles:
timestamp = arrow.utcnow().timestamp
res = json.loads(open("results.json").read())[:]
values = dict([[i["username"],i["followers"]] for i in res])
#print values[values.keys()[0]], values.keys()[0]
ssdb.multi_zset("instagram-followers", **values)
ssdb.multi_zset("test-instagram-followers", **values)
values = dict([[i["username"], timestamp] for i in res])
ssdb.multi_zset("ig-last-crawled", **values)
ssdb.multi_zset("ig-last-updated", **values)
values = dict([[i[_id], i["username"]] for i in res])
ssdb.multi_hset("ig-username-id", **values)
for c, user in enumerate(json.loads(open("results.json").read())[:]):
print c
un = user["username"]
cols = ["profile_pic_url","full_name","followers","following","username"]
vals = [user[c] for c in cols]
info = dict(zip(cols, vals))
#Information
def __init__(self, table, host="localhost", port=8888, **kwargs):
self.name = table
pool = BlockingConnectionPool(host=host, port=port, **kwargs)
self.__conn = SSDB(connection_pool=pool)
#!/usr/bin/python
from ssdb import SSDB
import json
import time
import os
from os import listdir
from os.path import isfile, join
from bs4 import BeautifulSoup
import json
import subprocess
ssdb = SSDB(host=os.environ['SSDB_HOST'], port=os.environ['SSDB_PORT'])
class Instagram:
def _profile(self, body):
bs = BeautifulSoup(body,"html.parser")
#lol = bs.find_all("script")[-4].text[21:]
lol = bs.find_all("script")[-5].text[21:]
lol = json.loads(lol[:-1])
user = lol["entry_data"]["ProfilePage"][0]["user"]
#pics = user["media"]["nodes"]
user["_id"] = user["id"]
del user["id"]
user["followers"] = user["followed_by"]["count"]
user["following"] = user["follows"]["count"]
user["picture-count"] = user["media"]["count"]
del user["media"]
del user["follows"]
del user["followed_by"]
def __init__(self, host, port, table_name, **args):
self.queue_name = table_name
pool = BlockingConnectionPool(host=host, port=port, **args)
self.ssdb = SSDB(connection_pool=pool)
print 'success init ssdb connection'
#encoding=utf8
import time
import redis
from ssdb import SSDB
rclient = redis.Redis(host="*", port=36379, password='******')
rpipeline = rclient.pipeline()
pipe_time = time.time()
for x in range(10000):
rpipeline.lpop('task_list_test01')
#rpipeline.lrange('task_list_test01',0,10000)
records = rpipeline.execute()
print('redis total time:', time.time() - pipe_time)
sclient = SSDB('*', port=46379)
# sclient = SSDB(host='18.196.*.*',port=46379,)
sclient.execute_command('auth xxxxxxx')
i = 1
begin_time = time.time()
sclient.qpush_front('task_list_sorted', records)
print(time.time() - begin_time)
exit(0)
for record in records:
# sclient.zset('task_list_sorted',record,i)
sclient.qpush_front('task_list_sorted', record)
i += 1
print(time.time() - begin_time)
#------------------------------------
#postgresql连接代码
class SsdbClient(object):
"""
SSDB client
SSDB中代理存放的容器为hash：
原始代理存放在name为raw_proxy的hash中，key为代理的ip:port，value为None,以后扩展可能会加入代理属性；
验证后供flask使用的代理存放在name为useful_proxy_queue的hash中，key为代理的ip:port，value为None,以后扩展可能会加入代理属性；
"""
def __init__(self, name, host, port):
"""
init
:param name: hash name
:param host: ssdb host
:param port: ssdb port
:return:
"""
self.name = name
self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=host, port=port))
def get(self):
"""
get an item
从useful_proxy_queue随机获取一个可用代理, 使用前需要调用changeTable("useful_proxy_queue")
:return:
"""
values = self.__conn.hgetall(name=self.name)
return random.choice(values.keys()) if values else None
def put(self, value):
"""
put an item
将代理放入hash, 使用changeTable指定hash name
:param value:
:return:
"""
value = json.dump(value, ensure_ascii=False).encode('utf-8') if isinstance(value, (dict, list)) else value
return self.__conn.hset(self.name, value, None)
def pop(self):
"""
pop an item
弹出一个代理， 使用changeTable指定hash name
:return:
"""
key = self.get()
if key:
self.__conn.hdel(self.name, key)
return key
def delete(self, key):
"""
delete an item
:param key:
:return:
"""
self.__conn.hdel(self.name, key)
def getAll(self):
return self.__conn.hgetall(self.name).keys()
def changeTable(self, name):
self.name = name
class ssdb(storage):
def __init__(self, host, port, table_name, **args):
self.queue_name = table_name
pool = BlockingConnectionPool(host=host, port=port, **args)
self.ssdb = SSDB(connection_pool=pool)
print 'success init ssdb connection'
def save(self, data, **args):
"""
push data to ssdb queue back
:param data: data of dict type
:return:（None)
"""
while True:
try:
if isinstance(data, dict) or isinstance(data, list):
self.ssdb.qpush_back(self.queue_name, json.dumps(data))
else:
self.ssdb.qpush_back(self.queue_name, data)
break
except Exception as e:
print e
time.sleep(60)
def get(self, **args):
while True:
try:
data = self.ssdb.qpop_front(self.queue_name)
if data:
return data[0]
return data
except Exception as e:
print e
time.sleep(60)
# 切换队列
def select_queue(self, name):
self.queue_name = name
def size(self, **args):
return self.ssdb.qsize(self.queue_name)
def len(self, **args):
return self.ssdb.qsize(self.queue_name)
def put(self, data, **args):
"""
push data to ssdb queue back
:param data: data of dict type
:return:（None)
"""
while True:
try:
if isinstance(data, dict) or isinstance(data, list):
self.ssdb.qpush_back(self.queue_name, json.dumps(data))
else:
self.ssdb.qpush_back(self.queue_name, data)
break
except Exception as e:
print e
time.sleep(60)
def put_data_back(self, data):
return self.ssdb.qpush_back(self.queue_name, data)
def put_data_front(self, data):
return self.ssdb.qpush_front(self.queue_name, data)
def ssdb_put_zset(self, field):
#zset zset key score
while True:
try:
return self.ssdb.zset(self.queue_name, field)
except Exception as e:
print e
self.SSDB()
def ssdb_zexists(self, field):
while True:
try:
return self.ssdb.zexists(self.queue_name, field)
except Exception as e:
print e
self.SSDB()
def ssdb_zdel(self, field):
while True:
try:
return self.ssdb.zdel(self.queue_name, field)
except Exception as e:
print e
self.SSDB()
def __init__(self, table=None, host='localhost', port=8888, **kwargs):
self.table = table
pool = BlockingConnectionPool(host=host, port=port, **kwargs)
self.ssdb = SSDB(connection_pool=pool)
# -*- coding: utf-8 -*-
from ssdb import SSDB
if __name__ == '__main__':
r = SSDB(host='10.68.120.175',
port=7777,
password='******')
# r = SSDB.from_url(url='ssdb://:[email protected]:7777')
# **Server**
# b = r.auth('11111111111111111111111111111111')
# b=r.info('cmd')
# b= r.dbsize()
# b = r.flushdb()
# **Key Value**
# b=r.set('aa',11) # 1
# b=r.setx('aa',11,1000) # 1
# b = r.setnx('bds', 11) # 1/0
# b = r.expire('aaqe',1000) # 1/0
# b = r.ttl('bdasd') # int value
b = r.get('bdas') # value/none
# b = r.getset('bdadssf','22') # value/none
# b = r.delete('aa') # 1
# b = r.incr('aaaasdfsd',2) # int value
# b = r.exists('aaa') # true/false
# b = r.getbit('14aaasdf',11) # 1/0
# b = r.setbit('asfga',11112341, 1) # 1/0
# b = r.bitcount('aavba', 0,1) # 1/0
# b = r.countbit('aa',0,-1) # 1/0
# b = r.substr('bdadssf',0,-1) # value
# b = r.strlen('asfga') # int value
from ssdb import SSDB
import sys
#import cugbacm.proto.rank_pb2
ssdb_ip = "127.0.0.1"
ssdb_port = 6666
ssdb = SSDB(host=ssdb_ip, port=ssdb_port)
def GetContestRankListProto(contestID):
global ssdb
proto_str = ssdb.get(contestID)
return proto_str
def SetContestRankListProto(contestID, rank_list_proto_str):
global ssdb
try:
ssdb.set(contestID, rank_list_proto_str)
except:
pass
def InsertUserProblemStatus(userID, problemID, status):
#AC == 1 No_pass = 2 other = 0
global ssdb
value = "2"
if status == "Accepted":
value = "1"
st = str(ssdb.get(userID + '\t' + str(problemID)))
if st == "1":
return;
else:
ssdb.set(userID + '\t' + str(problemID), value)
from ssdb import SSDB
import sys
import cugbacm.proto.rank_pb2
ssdb_ip = "127.0.0.1"
ssdb_port = 6666
ssdb = SSDB(host=ssdb_ip, port=ssdb_port)
def GetContestRankListProto(contestID):
global ssdb
proto_str = ssdb.get(contestID)
return proto_str
def SetContestRankListProto(contestID, rank_list_proto_str):
global ssdb
try:
ssdb.set(contestID, rank_list_proto_str)
except:
pass
def InsertUserProblemStatus(userID, problemID, status):
#AC == 1 No_pass = 2 other = 0
global ssdb
value = 2
if status == "Accepted":
value = 1
st = str(ssdb.get(userID + '\t' + str(problemID)))
if st == "1":
return;
else:
ssdb.set(userID + '\t' + str(problemID), value)
# Get URLS from SSDB and turn into a text file called links
import subprocess
import os
from ssdb import SSDB
ssdb = SSDB(host='104.131.54.255', port=8888)
# get hostid
hostid = subprocess.check_output(['curl','http://ipinfo.io/ip']).strip()
limit = ssdb.hget("crawl","limit")
offset = ssdb.hget("crawl", hostid)
users = ssdb.zrange("instagram-users", int(offset), int(limit))
arr = users
f = open("links", "w")
f.write("\n".join(map(lambda x: str(x), arr)))
f.close()
#!/usr/bin/env python # -*- coding: utf-8 -*- """ @license : (C) Copyright 2013-2017, Easy doesnt enter into grown-up life. @Software: PyCharm @Project : appstore @Time : 2018/4/5 下午5:18 @Author : achilles_xushy @contact : [email protected] @Site : @File : db.py @desc : """ from ssdb import SSDB # 主业务 db = SSDB(host='127.0.0.1', port=8887, socket_timeout=600)
from ssdb import SSDB
import sys
#import cugbacm.proto.rank_pb2
ssdb_ip = "127.0.0.1"
ssdb_port = 6666
ssdb = SSDB(host=ssdb_ip, port=ssdb_port)
def GetContestRankListProto(contestID):
global ssdb
proto_str = ssdb.get(contestID)
return proto_str
def SetContestRankListProto(contestID, rank_list_proto_str):
global ssdb
try:
ssdb.set(contestID, rank_list_proto_str)
except:
pass
def InsertUserProblemStatus(userID, problemID, status):
#AC == 1 No_pass = 2 other = 0
global ssdb
value = "2"
if status == "Accepted":
value = "1"
st = str(ssdb.get(userID + '\t' + str(problemID)))
if st == "1":