Пример #1
0
class myssdb(object):

    def __init__(self, table=None, host='localhost', port=8888, **kwargs):
        self.table = table
        pool = BlockingConnectionPool(host=host, port=port, **kwargs)
        self.ssdb = SSDB(connection_pool=pool)


    # 队尾放入一条数据
    def put(self, data, **kwargs):
        length = 0
        if isinstance(data, dict) or isinstance(data, list):
            length = self.ssdb.qpush_back(self.table, json.dumps(data))
        else:
            length = self.ssdb.qpush_back(self.table, data)
        if length:
            self.ssdb.hincr('counter_' + self.table, time.strftime('%Y-%m-%d', time.localtime(float(time.time()))))
        return length


    # 队首取出一条数据
    def get(self, **kwargs):
        data = self.ssdb.qpop_front(self.table)
        if data:
            return data[0]
        return data
Пример #2
0
def quary():
    keyword = "weibo_拾笔不良_2017-05-08"

    try:
        client = SSDB(host='', port=8884)  #连接到服务器的SSDB数据库
        res = client.get(keyword)
    except Exception as e:
        print e.message
        results = []
    else:
        results = json.loads(res)
    finally:
        print len(results)

    for block in results:
        print "++++++++++++++"
        print "name:", block['name']
        " *** 重要 *** "
        print "content:", urllib.unquote(str(
            block['content']))  #首先将结果转换为字符串,然后对字符串进行url解码
        print "point:", block['point']

        comment_list = block['comment_list']
        for comment in comment_list:
            print "comment:", comment
Пример #3
0
class SSDBKV(object):
    def __init__(self,
                 host="127.0.0.1",
                 port=8888,
                 max_connections=10,
                 timeout=60):
        self.host = host
        self.port = port
        self.max_connections = max_connections
        self.timeout = timeout
        pool = BlockingConnectionPool(connection_class=Connection,
                                      max_connections=max_connections,
                                      timeout=timeout,
                                      host=host,
                                      port=port)
        self.ssdb = SSDB(connection_pool=pool)

    def set(self, key, value):
        return self.ssdb.set(key, value)

    def get(self, key):
        return self.ssdb.get(key)

    def delete(self, key):
        return self.ssdb.delete(key)

    def keys(self, name_start=0, name_end=0xFFFFFFFF, limit=10):
        return self.ssdb.keys(name_start, name_end, limit)

    def exists(self, key):
        return self.ssdb.exists(key)
Пример #4
0
def processString(arg):
    keyList = arg[0]
    redisHost = arg[1]
    redisPort = arg[2]
    ssdbHost = arg[3]
    ssdbPort = arg[4]

    if len(keyList) == 0:
        return
    try:
        #get data from redis
        data = {}
        r = redis.Redis(host=redisHost, port=redisPort)
        #get keys
        retList = r.mget(keyList)
        for i in range(len(retList)):
            if retList[i] != "(nil)":
                data[keyList[i]] = retList[i]
        #put keys
        print "get redis data size: %d" % (len(data))
        #put data to ssdb
        s = SSDB(SSDB_HOST, SSDB_PORT)
        for key in data:
            s.setx(key, data[key], EX_TIME)
            print "set key: %s" % (key)
    except Exception, e:
        print e.message
Пример #5
0
 def ssdb_info(self):
     try:
         ssdb_cli = SSDB(host=self.addr, port=self.port)
         ssdb_info_list = ssdb_cli.execute_command("info")[2:-2]
         ssdb_info = self.process_ssdb_info_simple(ssdb_info_list)
     except Exception as e:
         self.logger.error(e)
         ssdb_info = {}
     return ssdb_info
Пример #6
0
 def __init__(self, name, host, port):
     """
     init
     :param name: hash name
     :param host: ssdb host
     :param port: ssdb port
     :return:
     """
     self.name = name
     self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=host, port=port))
Пример #7
0
 def __init__(self, queue_name, host, port, max_connections=10, timeout=60):
     self.queue_name = queue_name
     self.host = host
     self.port = port
     self.max_connections = max_connections
     self.timeout = timeout
     pool = BlockingConnectionPool(connection_class=Connection,
                                   max_connections=max_connections,
                                   timeout=timeout,
                                   host=host,
                                   port=port)
     self.ssdb = SSDB(connection_pool=pool)
Пример #8
0
    def __init__(self, id):
        self.check_param()

        self.mapId_list = []
        self.setId_list = []

        for param in FLUSH_MAP_ID:
            self.mapId_list.append(param + id)

        for param in FLUSH_SET_ID:
            self.setId_list.append(param + id)

        self.ssdb = SSDB(host=SOURCE_IP, port=SOURCE_PORT)
Пример #9
0
def getIP():
    client = SSDB(host='10.141.5.89', port=8884)
    keyword = "proxy"
    res = client.get(keyword)

    addr_list = list()
    ip_port_list = res.split(",")[0:-2]
    """拆分出IP和Port"""
    for ip_port in ip_port_list:
        ip, port = re.split("\s+", ip_port)
        addr_list.append(ip + ":" + port)

    print "addr parse successful"
    return addr_list
Пример #10
0
 def __init__(self, set_name, host, port, max_connections=2, timeout=60):
     """数据库初始化"""
     self.set_name = set_name
     self.host = host
     self.port = port
     self.max_connections = max_connections
     self.timeout = timeout
     pool = BlockingConnectionPool(connection_class=Connection,
                                   max_connections=max_connections,
                                   timeout=timeout,
                                   host=host,
                                   port=port)
     self.ssdb = SSDB(connection_pool=pool)
     pass
Пример #11
0
def process(keyword):
    addr_list = tools.getIP()
    addr = addr_list[random.randint(0, len(addr_list)-1)]       # 随机选择一个代理IP
    """代理设置"""
    proxy = Proxy(
        {
            'proxy_type': ProxyType.MANUAL,
            'http_proxy': addr
        }
    )
    desired_capabilities = webdriver.DesiredCapabilities.PHANTOMJS
    proxy.add_to_capabilities(desired_capabilities)

    """1) 构造driver对象,并设置窗口尺寸"""
    driver = webdriver.PhantomJS(desired_capabilities=desired_capabilities)
    driver.maximize_window()

    """2) 通过driver对象,实例化Sina类"""
    sina = Sina(DRIVER=driver)

    """3) 实现登录,两种登录方式二选一"""
    sina.login2()
    # sina.login()

    """4) 搜索"""
    sina.search(keyword)

    """连接到SSDB数据库"""
    client = SSDB(host='', port=8884)

    """获取结果集"""
    results = list()
    page = 1
    while len(results) < 100:   #至少返回100条数据
        print "+++++++++++++++++++++++++++++++++++++++", "page", page
        block_list = sina.extract()
        results.extend(block_list)

        page += 1

        if sina.nextPage():
            continue
        else:
            break

    key = "weibo_"+keyword+"_"+tools.getTimeAsStr()
    value = json.dumps(results)
    client.set(key, value)

    print "ssdb save", key, len(results)
Пример #12
0
 def __init__(self,
              host="127.0.0.1",
              port=8888,
              max_connections=10,
              timeout=60):
     self.host = host
     self.port = port
     self.max_connections = max_connections
     self.timeout = timeout
     pool = BlockingConnectionPool(connection_class=Connection,
                                   max_connections=max_connections,
                                   timeout=timeout,
                                   host=host,
                                   port=port)
     self.ssdb = SSDB(connection_pool=pool)
Пример #13
0
class Gtoken():
    count = 0
    ssdb = SSDB(host='localhost', port=8888)
    while True:
        post_id = ssdb.get('raw' + str(count))
        dapet_komen(post_id)
        count += 1
Пример #14
0
class SsdbClient(object):
    """
    SSDB client
    """
    def __init__(self, name, host, port):
        """
        init
        :param name:
        :param host:
        :param port:
        :return:
        """
        self.name = name
        self.__conn = SSDB(
            connection_pool=BlockingConnectionPool(host=host, port=port))

    def get(self):
        """
        get an item
        :return:
        """
        values = self.__conn.hgetall(name=self.name)
        return random.choice(values.keys()) if values else None

    def put(self, value):
        """
        put an  item
        :param value:
        :return:
        """
        value = json.dump(value,
                          ensure_ascii=False).encode('utf-8') if isinstance(
                              value, (dict, list)) else value
        return self.__conn.hset(self.name, value, None)

    def pop(self):
        """
        pop an item
        :return:
        """
        key = self.get()
        if key:
            self.__conn.hdel(self.name, key)
        return key

    def delete(self, key):
        """
        delete an item
        :param key:
        :return:
        """
        self.__conn.hdel(self.name, key)

    def getAll(self):
        return self.__conn.hgetall(self.name).keys()

    def changeTable(self, name):
        self.name = name
Пример #15
0
 def __init__(self, name, host, port):
     """
     init
     :param name: hash name
     :param host: ssdb host
     :param port: ssdb port
     :return:
     """
     self.name = name
     self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=host, port=port))
Пример #16
0
class SSDBHashMap(object, SSDBBase):
    def __init__(self,
                 hashmap_name,
                 host="127.0.0.1",
                 port=8888,
                 max_connections=10,
                 timeout=60):
        self.hashmap_name = hashmap_name
        self.host = host
        self.port = port
        self.max_connections = max_connections
        self.timeout = timeout
        pool = BlockingConnectionPool(connection_class=Connection,
                                      max_connections=max_connections,
                                      timeout=timeout,
                                      host=host,
                                      port=port)
        self.ssdb = SSDB(connection_pool=pool)
        SSDBBase.__init__(self, self.ssdb)

    def set(self, key, value):
        return self.ssdb.hset(self.hashmap_name, key, value)

    def get(self, key):
        return self.ssdb.hget(self.hashmap_name, key)

    def delete(self, key):
        return self.ssdb.hdel(self.hashmap_name, key)

    def keys(self, name_start="", name_end="", limit=10):
        return self.ssdb.hkeys(self.hashmap_name, name_start, name_end, limit)

    def exists(self, key):
        return self.ssdb.hexists(self.hashmap_name, key)

    def size(self):
        return self.ssdb.hsize(self.hashmap_name)

    def list(self, name_start="", name_end="", limit=10):
        #列出名字处于区间 (name_start, name_end] 的 hashmap
        return self.ssdb.hlist(name_start, name_end, limit)

    def scan(self, key_start, key_end="", limit=10):
        return self.ssdb.hscan(self.hashmap_name, key_start, key_end, limit)

    def clear(self):
        return self.ssdb.hclear(self.hashmap_name)
Пример #17
0
def setExpire(keyList, redisType, extime):
    if redisType not in ["string", "list", "set", "hash", "sortedset"
                         ] or len(keyList) == 0:
        print "key type error! type: %s" % (redisType)
        return

    data = {}
    data[EX_KEY] = {}
    for key in keyList:
        stringKey = "%s_%s" % (redisType, key)
        data[EX_KEY][stringKey] = int(time.time()) + extime
        print "expire key: %s" % (stringKey)
    print "get expire key size: %d" % (len(data))
    #print data
    #put data to ssdb
    s = SSDB(SSDB_HOST, SSDB_PORT)
    for key in data:
        s.multi_zset(key, **data[key])
        print "zadd key: %s" % (key)
    print "set expire key finish"
Пример #18
0
class myssdb_hash(object):
    def __init__(self, table, host="localhost", port=8888, **kwargs):
        self.name = table
        pool = BlockingConnectionPool(host=host, port=port, **kwargs)
        self.__conn = SSDB(connection_pool=pool)

    # put a data
    def put(self, value, **kwargs):
        if isinstance(value, dict):
            key = value.get("_id", str(uuid.uuid1()))
        elif isinstance(value, basestring):
            try:
                tmp = json.loads(value)
                if isinstance(tmp, dict):
                    key = tmp.get("_id", str(uuid.uuid1()))
                else:
                    key = str(uuid.uuid1())
            except:
                key = str(uuid.uuid1())
        else:
            key = str(uuid.uuid1())
        status = self.__conn.hset(self.name, key, value if isinstance(value, basestring) else json.dumps(value))
        self.__conn.hincr("counter_" + self.name, time.strftime("%Y-%m-%d", time.localtime(float(time.time()))))
        return status

    # get a data
    def get(self, **kwargs):
        tmp = self.__conn.hscan(self.name, "", "", limit=1)
        if not tmp:
            return None
        try:
            self.__conn.hdel(self.name, tmp.keys()[0])
        except Exception as e:
            print str(e)
            return None
        return tmp.values()[0]

    # table size
    def size(self, *args, **kwargs):
        return self.__conn.hsize(self.name)

    def __list(self):
        start_key = ""
        for i in xrange(0, self.size(), 1000):
            tmp = self.__conn.hscan(self.name, start_key, "", limit=1000)
            start_key = tmp.keys()[-1]
            for value in tmp.values():
                yield value

    def get_values(self):
        return self.__list()
Пример #19
0
class QueueSSDB(QueueBase.QueueBase):
    def __init__(self, name, host='localhost', port=8888, **kwargs):
        QueueBase.QueueBase.__init__(self, name, host, port)
        self.__conn = SSDB(connection_pool=BlockingConnectionPool(
            host=self.host, port=self.port))

    @QueueBase.catch
    def put(self, value, *args, **kwargs):
        return self.__conn.qpush_back(
            self.name,
            json.dumps(value)
            if isinstance(value, dict) or isinstance(value, list) else value)

    @QueueBase.catch
    def get(self, *args, **kwargs):
        value = self.__conn.qpop_front(self.name)
        return value[0] if value else value

    @QueueBase.catch
    def size(self, *args, **kwargs):
        return self.__conn.qsize(self.name)
Пример #20
0
def processList(arg):
    keyList = arg[0]
    redisHost = arg[1]
    redisPort = arg[2]
    ssdbHost = arg[3]
    ssdbPort = arg[4]

    if len(keyList) == 0:
        return
    try:
        #get data from redis
        data = {}
        r = redis.Redis(host=redisHost, port=redisPort)
        pipeRedis = r.pipeline()
        #get keys
        for item in keyList:
            pipeRedis.lrange(item, 0, -1)
        retList = pipeRedis.execute()
        for i in range(len(retList)):
            if len(retList[i]) > 0:
                data[keyList[i]] = retList[i]
        #put keys
        print "get redis data size: %d" % (len(data))
        #print data
        #put data to ssdb
        s = SSDB(SSDB_HOST, SSDB_PORT)
        for key in data:
            s.qclear(key)
            s.qpush_back(key, *data[key])
            print "lpush key: %s" % (key)
        #set expire
        setExpire(keyList, "list", EX_TIME)
    except Exception, e:
        print e.message
Пример #21
0
class QueueHash(QueueBase.QueueBase):
    def __init__(self, name, host='localhost', port=8888, **kwargs):
        QueueBase.QueueBase.__init__(self, name, host, port)
        self.__conn = SSDB(connection_pool=BlockingConnectionPool(
            host=self.host, port=self.port))

    @QueueBase.catch
    def put(self, value, *args, **kwargs):
        if isinstance(value, dict):
            key = value.get('_id', str(uuid.uuid1()))
        elif isinstance(value, basestring):
            try:
                tmp = json.loads(value)
                if isinstance(tmp, dict):
                    key = tmp.get('_id', str(uuid.uuid1()))
                else:
                    key = str(uuid.uuid1())
            except:
                key = str(uuid.uuid1())
        else:
            key = str(uuid.uuid1())
        return self.__conn.hset(
            self.name, key, value if isinstance(value, basestring) else
            json.dumps(value, ensure_ascii=False).encode('utf-8'))

    @QueueBase.catch
    def get(self, *args, **kwargs):
        tmp = self.__conn.hscan(self.name, '', '', limit=1)
        if not tmp:
            return None
        try:
            self.__conn.hdel(self.name, tmp.keys()[0])
        except Exception as e:
            print str(e)
            return None
        return tmp.values()[0]

    @QueueBase.catch
    def size(self, *args, **kwargs):
        return self.__conn.hsize(self.name)
Пример #22
0
class Clear(object):
    def __init__(self, id):
        self.check_param()

        self.mapId_list = []
        self.setId_list = []

        for param in FLUSH_MAP_ID:
            self.mapId_list.append(param + id)

        for param in FLUSH_SET_ID:
            self.setId_list.append(param + id)

        self.ssdb = SSDB(host=SOURCE_IP, port=SOURCE_PORT)

    def check_param(self):
        if CHARACTER_SET == "" or SOURCE_IP == "" or SOURCE_PORT == "":
            print("Check Clear Parameters!")
            sys.exit()

        for param in FLUSH_MAP_ID:
            if param == "":
                print("Check Clear Parameters!")

        for param in FLUSH_SET_ID:
            if param == "":
                print("Check Clear Parameters!")

    def show_config(self):
        print("")
        print("SOURCE_IP = %s" % SOURCE_IP)
        print("SOURCE_PORT = %s" % SOURCE_PORT)
        print("")

        index = 1
        for param in self.mapId_list:
            print("mapId%s = %s,Size = %s" %
                  (index, param, self.ssdb.hsize(param)))
            index = index + 1
        print("")

        index = 1
        for param in self.setId_list:
            print("setId%s = %s,Size = %s" %
                  (index, param, self.ssdb.zsize(param)))
            index = index + 1
        print("")

    def clear_data(self):

        for param in self.mapId_list:
            self.ssdb.hclear(param)
            print("Clear %s Success!" % (param))

        for param in self.setId_list:
            self.ssdb.zclear(param)
            print("Clear %s Success!" % (param))

        print("Clear SSDB Success!")
Пример #23
0
    def ping_ssdb(self):
        ssdb_is_alive = 0
        try:
            ssdb_cli = SSDB(host=self.addr, port=self.port)
            ping_res = ssdb_cli.execute_command("ping")
            if ping_res:
                ssdb_is_alive = 1
        except Exception as e:
            self.logger.error(e)
        if ssdb_is_alive == 0:  # If ssdb is dead, update the alive metrice here.
            ssdb_alive_data = [{
                "endpoint": self.addr,
                "metric": "ssdb_alive",
                "tags": self.tags,
                "timestamp": upload_ts,
                "value": ssdb_is_alive,
                "step": 60,
                "counterType": "GAUGE"
            }]

            r = requests.post(falcon_client, data=json.dumps(ssdb_alive_data))
            self.logger.debug(r.text)
        return ssdb_is_alive
Пример #24
0
class SSDBZset(object, SSDBBase):
    def __init__(self,
                 zset_name,
                 host="127.0.0.1",
                 port=8888,
                 max_connections=10,
                 timeout=60):
        self.zset_name = zset_name
        self.host = host
        self.port = port
        self.max_connections = max_connections
        self.timeout = timeout
        pool = BlockingConnectionPool(connection_class=Connection,
                                      max_connections=max_connections,
                                      timeout=timeout,
                                      host=host,
                                      port=port)
        self.ssdb = SSDB(connection_pool=pool)
        SSDBBase.__init__(self, self.ssdb)

    def set(self, key, value, score):
        return self.ssdb.zset(self.zset_name, key, score)

    def get(self, key):
        return self.ssdb.zget(self.zset_name, key)

    def delete(self, key):
        return self.ssdb.zdel(self.zset_name, key)

    def keys(self, key_start="", score_start="", score_end="", limit=10):
        return self.ssdb.zkeys(self.zset_name, key_start, score_start,
                               score_end, limit)

    def exists(self, key):
        return self.ssdb.zexists(self.zset_name, key)

    def size(self):
        return self.ssdb.zsize(self.zset_name)

    def list(self, name_start="", name_end="", limit=10):
        #列出名字处于区间 (name_start, name_end] 的 hashmap
        return self.ssdb.zlist(name_start, name_end, limit)

    def clear(self):
        return self.ssdb.zclear(self.zset_name)
Пример #25
0
class ssdb_proxy(object):
    def __init__(self, set_name, host, port, max_connections=2, timeout=60):
        """数据库初始化"""
        self.set_name = set_name
        self.host = host
        self.port = port
        self.max_connections = max_connections
        self.timeout = timeout
        pool = BlockingConnectionPool(connection_class=Connection,
                                      max_connections=max_connections,
                                      timeout=timeout,
                                      host=host,
                                      port=port)
        self.ssdb = SSDB(connection_pool=pool)
        pass

    def hset(self, key, value):
        """添加集合"""
        return self.ssdb.hset(self.set_name, key, value)

    def hget(self, key):
        """获取key所对应的值"""
        return self.ssdb.hget(self.set_name, key)

    def hgetall(self):
        """获取所有的数据"""
        return self.ssdb.hgetall(self.set_name)

    def hdel(self, key):
        """删除数据"""
        return self.ssdb.hdel(self.set_name, key)

    def hexists(self, key):
        """判断Key是否存在"""
        return self.ssdb.hexists(self.set_name, key)

    def size(self):
        """获取结合的大小"""
        return self.ssdb.hsize(self.set_name)

    def clean(self):
        """清楚所有数据"""
        return self.ssdb.hclear(self.set_name)
Пример #26
0
def processSet(arg):
    keyList = arg[0]
    redisHost = arg[1]
    redisPort = arg[2]
    ssdbHost = arg[3]
    ssdbPort = arg[4]

    if len(keyList) == 0:
        return
    try:
        #get data from redis
        data = {}
        r = redis.Redis(host=redisHost, port=redisPort)
        pipeRedis = r.pipeline()
        #get keys
        for item in keyList:
            pipeRedis.smembers(item)
        retList = pipeRedis.execute()
        #print retList
        for i in range(len(retList)):
            if len(retList[i]) > 0:
                data[keyList[i]] = {}
                for item in retList[i]:
                    data[keyList[i]][item] = 0
        #put keys
        print "get redis data size: %d" % (len(data))
        #print data
        #put data to ssdb
        s = SSDB(SSDB_HOST, SSDB_PORT)
        for key in data:
            s.zclear(key)
            s.multi_zset(key, **data[key])
            print "zadd key: %s" % (key)

        #set expire
        setExpire(keyList, "sortedset", EX_TIME)
    except Exception, e:
        print e.message
Пример #27
0
 def __init__(self, host='localhost', port=8888, **kwargs):
     QueueBase.QueueBase.__init__(self, host, port)
     self.__conn = SSDB(connection_pool=BlockingConnectionPool(
         host=self.host, port=self.port))
     self.name = kwargs.get('keyName', 'default')
Пример #28
0
import arrow
from ssdb import SSDB
import json
ssdb = SSDB(host='104.131.54.255', port=8888)

onlyfiles = [f for f in listdir(".") if isfile(join(".", f))]
onlyfiles = [i for i in onlyfiles if "results-" in i]

for file_name in onlyfiles:
        timestamp = arrow.utcnow().timestamp
        res = json.loads(open("results.json").read())[:]
        values = dict([[i["username"],i["followers"]] for i in res])
        #print values[values.keys()[0]], values.keys()[0]
        ssdb.multi_zset("instagram-followers", **values)
        ssdb.multi_zset("test-instagram-followers", **values)
        
        values = dict([[i["username"], timestamp] for i in res])
        ssdb.multi_zset("ig-last-crawled", **values)
        ssdb.multi_zset("ig-last-updated", **values)
        
        values = dict([[i[_id], i["username"]] for i in res])
        ssdb.multi_hset("ig-username-id", **values)

        for c, user in enumerate(json.loads(open("results.json").read())[:]):
                print c
                un = user["username"]
                cols = ["profile_pic_url","full_name","followers","following","username"]
                vals = [user[c] for c in cols]
                info = dict(zip(cols, vals))
        
                #Information
Пример #29
0
 def __init__(self, table, host="localhost", port=8888, **kwargs):
     self.name = table
     pool = BlockingConnectionPool(host=host, port=port, **kwargs)
     self.__conn = SSDB(connection_pool=pool)
Пример #30
0
#!/usr/bin/python

from ssdb import SSDB
import json
import time
import os
from os import listdir
from os.path import isfile, join
from bs4 import BeautifulSoup
import json
import subprocess
ssdb = SSDB(host=os.environ['SSDB_HOST'], port=os.environ['SSDB_PORT'])

class Instagram:
    def _profile(self, body):
        bs = BeautifulSoup(body,"html.parser")

        #lol = bs.find_all("script")[-4].text[21:]
        lol = bs.find_all("script")[-5].text[21:]
        lol = json.loads(lol[:-1])
        user = lol["entry_data"]["ProfilePage"][0]["user"]

        #pics = user["media"]["nodes"]
        user["_id"] = user["id"]
        del user["id"]
        user["followers"] = user["followed_by"]["count"]
        user["following"] = user["follows"]["count"]
        user["picture-count"] = user["media"]["count"]
        del user["media"]
        del user["follows"]
        del user["followed_by"]
Пример #31
0
 def __init__(self, host, port, table_name, **args):
     self.queue_name = table_name
     pool = BlockingConnectionPool(host=host, port=port, **args)
     self.ssdb = SSDB(connection_pool=pool)
     print 'success init ssdb connection'
Пример #32
0
#encoding=utf8
import time
import redis
from ssdb import SSDB

rclient = redis.Redis(host="*", port=36379, password='******')
rpipeline = rclient.pipeline()

pipe_time = time.time()
for x in range(10000):
    rpipeline.lpop('task_list_test01')
#rpipeline.lrange('task_list_test01',0,10000)
records = rpipeline.execute()
print('redis total time:', time.time() - pipe_time)

sclient = SSDB('*', port=46379)
# sclient = SSDB(host='18.196.*.*',port=46379,)
sclient.execute_command('auth xxxxxxx')
i = 1
begin_time = time.time()
sclient.qpush_front('task_list_sorted', records)
print(time.time() - begin_time)

exit(0)
for record in records:
    # sclient.zset('task_list_sorted',record,i)
    sclient.qpush_front('task_list_sorted', record)
    i += 1
print(time.time() - begin_time)
#------------------------------------
#postgresql连接代码
Пример #33
0
class SsdbClient(object):
    """
    SSDB client

    SSDB中代理存放的容器为hash:
        原始代理存放在name为raw_proxy的hash中,key为代理的ip:port,value为None,以后扩展可能会加入代理属性;
        验证后供flask使用的代理存放在name为useful_proxy_queue的hash中,key为代理的ip:port,value为None,以后扩展可能会加入代理属性;

    """

    def __init__(self, name, host, port):
        """
        init
        :param name: hash name
        :param host: ssdb host
        :param port: ssdb port
        :return:
        """
        self.name = name
        self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=host, port=port))

    def get(self):
        """
        get an item

        从useful_proxy_queue随机获取一个可用代理, 使用前需要调用changeTable("useful_proxy_queue")
        :return:
        """
        values = self.__conn.hgetall(name=self.name)
        return random.choice(values.keys()) if values else None

    def put(self, value):
        """
        put an  item

        将代理放入hash, 使用changeTable指定hash name
        :param value:
        :return:
        """
        value = json.dump(value, ensure_ascii=False).encode('utf-8') if isinstance(value, (dict, list)) else value
        return self.__conn.hset(self.name, value, None)

    def pop(self):
        """
        pop an item

        弹出一个代理, 使用changeTable指定hash name
        :return:
        """
        key = self.get()
        if key:
            self.__conn.hdel(self.name, key)
        return key

    def delete(self, key):
        """
        delete an item
        :param key:
        :return:
        """
        self.__conn.hdel(self.name, key)

    def getAll(self):
        return self.__conn.hgetall(self.name).keys()

    def changeTable(self, name):
        self.name = name
Пример #34
0
class ssdb(storage):
    def __init__(self, host, port, table_name, **args):
        self.queue_name = table_name
        pool = BlockingConnectionPool(host=host, port=port, **args)
        self.ssdb = SSDB(connection_pool=pool)
        print 'success init ssdb connection'

    def save(self, data, **args):
        """
        push data to ssdb queue back
        :param data: data of dict type
        :return:(None)
        """
        while True:
            try:
                if isinstance(data, dict) or isinstance(data, list):
                    self.ssdb.qpush_back(self.queue_name, json.dumps(data))
                else:
                    self.ssdb.qpush_back(self.queue_name, data)
                break
            except Exception as e:
                print e
                time.sleep(60)

    def get(self, **args):
        while True:
            try:
                data = self.ssdb.qpop_front(self.queue_name)
                if data:
                    return data[0]
                return data
            except Exception as e:
                print e
                time.sleep(60)

    # 切换队列
    def select_queue(self, name):
        self.queue_name = name

    def size(self, **args):
        return self.ssdb.qsize(self.queue_name)

    def len(self, **args):
        return self.ssdb.qsize(self.queue_name)

    def put(self, data, **args):
        """
        push data to ssdb queue back
        :param data: data of dict type
        :return:(None)
        """
        while True:
            try:
                if isinstance(data, dict) or isinstance(data, list):
                    self.ssdb.qpush_back(self.queue_name, json.dumps(data))
                else:
                    self.ssdb.qpush_back(self.queue_name, data)
                break
            except Exception as e:
                print e
                time.sleep(60)

    def put_data_back(self, data):
        return self.ssdb.qpush_back(self.queue_name, data)

    def put_data_front(self, data):
        return self.ssdb.qpush_front(self.queue_name, data)

    def ssdb_put_zset(self, field):
        #zset zset key score
        while True:
            try:
                return self.ssdb.zset(self.queue_name, field)
            except Exception as e:
                print e
                self.SSDB()

    def ssdb_zexists(self, field):
        while True:
            try:
                return self.ssdb.zexists(self.queue_name, field)
            except Exception as e:
                print e
                self.SSDB()

    def ssdb_zdel(self, field):
        while True:
            try:
                return self.ssdb.zdel(self.queue_name, field)
            except Exception as e:
                print e
                self.SSDB()
Пример #35
0
 def __init__(self, table=None, host='localhost', port=8888, **kwargs):
     self.table = table
     pool = BlockingConnectionPool(host=host, port=port, **kwargs)
     self.ssdb = SSDB(connection_pool=pool)
Пример #36
0
# -*- coding: utf-8 -*-

from ssdb import SSDB

if __name__ == '__main__':
    r = SSDB(host='10.68.120.175',
             port=7777,
             password='******')
    # r = SSDB.from_url(url='ssdb://:[email protected]:7777')
    # **Server**
    # b = r.auth('11111111111111111111111111111111')
    # b=r.info('cmd')
    # b= r.dbsize()
    # b = r.flushdb()
    # **Key Value**
    # b=r.set('aa',11)                          # 1
    # b=r.setx('aa',11,1000)                    # 1
    # b = r.setnx('bds', 11)                    # 1/0
    # b = r.expire('aaqe',1000)                 # 1/0
    # b = r.ttl('bdasd')                        # int value
    b = r.get('bdas')  # value/none
    # b = r.getset('bdadssf','22')              # value/none
    # b = r.delete('aa')                        # 1
    # b = r.incr('aaaasdfsd',2)                 # int value
    # b = r.exists('aaa')                       # true/false
    # b = r.getbit('14aaasdf',11)               # 1/0
    # b = r.setbit('asfga',11112341, 1)         # 1/0
    # b = r.bitcount('aavba', 0,1)              # 1/0
    # b = r.countbit('aa',0,-1)                 # 1/0
    # b = r.substr('bdadssf',0,-1)              # value
    # b = r.strlen('asfga')                     # int value
Пример #37
0
from ssdb import SSDB
import sys
#import cugbacm.proto.rank_pb2

ssdb_ip = "127.0.0.1"
ssdb_port = 6666
ssdb = SSDB(host=ssdb_ip, port=ssdb_port)

def GetContestRankListProto(contestID):
  global ssdb
  proto_str = ssdb.get(contestID)
  return proto_str

def SetContestRankListProto(contestID, rank_list_proto_str):
  global ssdb
  try:
    ssdb.set(contestID, rank_list_proto_str)
  except:
    pass
def InsertUserProblemStatus(userID, problemID, status):
  #AC == 1  No_pass = 2 other = 0
  global ssdb
  value = "2"
  if status == "Accepted":
    value = "1"
  st = str(ssdb.get(userID + '\t' + str(problemID)))
  if st == "1":
    return;
  else:
    ssdb.set(userID + '\t' + str(problemID), value)
Пример #38
0
from ssdb import SSDB
import sys
import cugbacm.proto.rank_pb2

ssdb_ip = "127.0.0.1"
ssdb_port = 6666
ssdb = SSDB(host=ssdb_ip, port=ssdb_port)

def GetContestRankListProto(contestID):
  global ssdb
  proto_str = ssdb.get(contestID)
  return proto_str

def SetContestRankListProto(contestID, rank_list_proto_str):
  global ssdb
  try:
    ssdb.set(contestID, rank_list_proto_str)
  except:
    pass
def InsertUserProblemStatus(userID, problemID, status):
  #AC == 1  No_pass = 2 other = 0
  global ssdb
  value = 2
  if status == "Accepted":
    value = 1
  st = str(ssdb.get(userID + '\t' + str(problemID)))
  if st == "1":
    return;
  else:
    ssdb.set(userID + '\t' + str(problemID), value)
Пример #39
0
# Get URLS from SSDB and turn into a text file called links
import subprocess
import os

from ssdb import SSDB
ssdb = SSDB(host='104.131.54.255', port=8888)

# get hostid
hostid = subprocess.check_output(['curl','http://ipinfo.io/ip']).strip()

limit = ssdb.hget("crawl","limit")
offset = ssdb.hget("crawl", hostid)

users = ssdb.zrange("instagram-users", int(offset), int(limit))
arr = users
 
f = open("links", "w")
f.write("\n".join(map(lambda x: str(x), arr)))
f.close()
Пример #40
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@license : (C) Copyright 2013-2017, Easy doesnt enter into grown-up life.
@Software: PyCharm
@Project : appstore
@Time : 2018/4/5 下午5:18
@Author : achilles_xushy
@contact : [email protected]
@Site :
@File : db.py
@desc :
"""

from ssdb import SSDB

# 主业务
db = SSDB(host='127.0.0.1', port=8887, socket_timeout=600)
Пример #41
0
from ssdb import SSDB
import sys
#import cugbacm.proto.rank_pb2

ssdb_ip = "127.0.0.1"
ssdb_port = 6666
ssdb = SSDB(host=ssdb_ip, port=ssdb_port)


def GetContestRankListProto(contestID):
    global ssdb
    proto_str = ssdb.get(contestID)
    return proto_str


def SetContestRankListProto(contestID, rank_list_proto_str):
    global ssdb
    try:
        ssdb.set(contestID, rank_list_proto_str)
    except:
        pass


def InsertUserProblemStatus(userID, problemID, status):
    #AC == 1  No_pass = 2 other = 0
    global ssdb
    value = "2"
    if status == "Accepted":
        value = "1"
    st = str(ssdb.get(userID + '\t' + str(problemID)))
    if st == "1":