Beispiel #1
0
class QueueSSDB(QueueBase.QueueBase):
    def __init__(self, name, host='localhost', port=8888, **kwargs):
        QueueBase.QueueBase.__init__(self, name, host, port)
        self.__conn = SSDB(connection_pool=BlockingConnectionPool(
            host=self.host, port=self.port))

    @QueueBase.catch
    def put(self, value, *args, **kwargs):
        return self.__conn.qpush_back(
            self.name,
            json.dumps(value)
            if isinstance(value, dict) or isinstance(value, list) else value)

    @QueueBase.catch
    def get(self, *args, **kwargs):
        value = self.__conn.qpop_front(self.name)
        return value[0] if value else value

    @QueueBase.catch
    def size(self, *args, **kwargs):
        return self.__conn.qsize(self.name)
Beispiel #2
0
class ssdb(storage):
    def __init__(self, host, port, table_name, **args):
        self.queue_name = table_name
        pool = BlockingConnectionPool(host=host, port=port, **args)
        self.ssdb = SSDB(connection_pool=pool)
        print 'success init ssdb connection'

    def save(self, data, **args):
        """
        push data to ssdb queue back
        :param data: data of dict type
        :return:(None)
        """
        while True:
            try:
                if isinstance(data, dict) or isinstance(data, list):
                    self.ssdb.qpush_back(self.queue_name, json.dumps(data))
                else:
                    self.ssdb.qpush_back(self.queue_name, data)
                break
            except Exception as e:
                print e
                time.sleep(60)

    def get(self, **args):
        while True:
            try:
                data = self.ssdb.qpop_front(self.queue_name)
                if data:
                    return data[0]
                return data
            except Exception as e:
                print e
                time.sleep(60)

    # 切换队列
    def select_queue(self, name):
        self.queue_name = name

    def size(self, **args):
        return self.ssdb.qsize(self.queue_name)

    def len(self, **args):
        return self.ssdb.qsize(self.queue_name)

    def put(self, data, **args):
        """
        push data to ssdb queue back
        :param data: data of dict type
        :return:(None)
        """
        while True:
            try:
                if isinstance(data, dict) or isinstance(data, list):
                    self.ssdb.qpush_back(self.queue_name, json.dumps(data))
                else:
                    self.ssdb.qpush_back(self.queue_name, data)
                break
            except Exception as e:
                print e
                time.sleep(60)

    def put_data_back(self, data):
        return self.ssdb.qpush_back(self.queue_name, data)

    def put_data_front(self, data):
        return self.ssdb.qpush_front(self.queue_name, data)

    def ssdb_put_zset(self, field):
        #zset zset key score
        while True:
            try:
                return self.ssdb.zset(self.queue_name, field)
            except Exception as e:
                print e
                self.SSDB()

    def ssdb_zexists(self, field):
        while True:
            try:
                return self.ssdb.zexists(self.queue_name, field)
            except Exception as e:
                print e
                self.SSDB()

    def ssdb_zdel(self, field):
        while True:
            try:
                return self.ssdb.zdel(self.queue_name, field)
            except Exception as e:
                print e
                self.SSDB()
Beispiel #3
0
class SSDBQueue(object):
    def __init__(self, queue_name, host, port, max_connections=10, timeout=60):
        self.queue_name = queue_name
        self.host = host
        self.port = port
        self.max_connections = max_connections
        self.timeout = timeout
        pool = BlockingConnectionPool(connection_class=Connection,
                                      max_connections=max_connections,
                                      timeout=timeout,
                                      host=host,
                                      port=port)
        self.ssdb = SSDB(connection_pool=pool)

    def queue_size(self):
        return self.ssdb.qsize(self.queue_name)

    def clear_queue(self):
        return self.ssdb.qclear(self.queue_name)

    def empty(self):
        return self.ssdb.qsize(self.queue_name) == 0

    def put_data(self, data):
        return self.ssdb.qpush_back(self.queue_name, data)

    def get_data(self):
        return self.ssdb.qpop_front(self.queue_name)

    def put(self,
            data,
            sleep_time=10,
            retry=9,
            email_list=functions.mailto_list_ourselves):
        i = 0
        count = retry + 1
        ssdb_queue = None
        while True:
            try:
                if data != None and ssdb_queue != None:
                    if isinstance(data, dict):
                        return ssdb_queue.put_data(pickle.dumps(data))
                else:
                    if isinstance(data, dict):
                        return self.put_data(pickle.dumps(data))
            except Exception as e:
                print u'插入队列异常 %s' % exceputil.traceinfo(e)
                i += 1
                if i >= count:
                    #发邮件
                    functions.send_mail_old(
                        email_list, u"ssdb队列更新异常", u"错误信息%s \nqueue_name:%s" %
                        (exceputil.traceinfo(e), self.queue_name))
                    time.sleep(600)
                    i = 0
                else:
                    time.sleep(sleep_time)
                ssdb_queue = getSSDBQueuev2(self.queue_name, self.host,
                                            self.port, self.max_connections,
                                            self.timeout)

    def get(self,
            sleep_time=10,
            retry=9,
            email_list=functions.mailto_list_ourselves):
        i = 0
        count = retry + 1
        ssdb_queue = None
        while True:
            try:
                if ssdb_queue != None:
                    return ssdb_queue.get_data()
                else:
                    return self.get_data()
            except Exception as e:
                print u'查询数据异常 %s' % exceputil.traceinfo(e)
                i += 1
                if i >= count:
                    #发邮件
                    functions.send_mail_old(email_list, u"ssdb队列更新异常",
                                            u"错误信息%s" % exceputil.traceinfo(e))
                    time.sleep(600)
                    i = 0
                else:
                    time.sleep(sleep_time)
                ssdb_queue = getSSDBQueuev2(self.queue_name, self.host,
                                            self.port, self.max_connections,
                                            self.timeout)

    def size(self,
             sleep_time=10,
             retry=9,
             email_list=functions.mailto_list_ourselves):
        i = 0
        count = retry + 1
        ssdb_queue = None
        while True:
            try:
                if ssdb_queue != None:
                    return ssdb_queue.queue_size()
                else:
                    return self.queue_size()
            except Exception as e:
                print u'查询队列中数异常 %s' % exceputil.traceinfo(e)
                i += 1
                if i >= count:
                    #发邮件
                    functions.send_mail_old(email_list, u"ssdb队列更新异常",
                                            u"错误信息%s" % exceputil.traceinfo(e))
                    time.sleep(600)
                    i = 0
                else:
                    time.sleep(sleep_time)
                ssdb_queue = getSSDBQueuev2(self.queue_name, self.host,
                                            self.port, self.max_connections,
                                            self.timeout)

    def clear(self,
              sleep_time=10,
              retry=9,
              email_list=functions.mailto_list_ourselves):
        i = 0
        count = retry + 1
        ssdb_queue = None
        while True:
            try:
                if ssdb_queue != None:
                    return ssdb_queue.clear_queue()
                else:
                    return self.clear_queue()
            except Exception as e:
                print u'查询队列中数异常 %s' % exceputil.traceinfo(e)
                i += 1
                if i >= count:
                    #发邮件
                    functions.send_mail_old(email_list, u"ssdb队列更新异常",
                                            u"错误信息%s" % exceputil.traceinfo(e))
                    time.sleep(600)
                    i = 0
                else:
                    time.sleep(sleep_time)
                ssdb_queue = getSSDBQueuev2(self.queue_name, self.host,
                                            self.port, self.max_connections,
                                            self.timeout)
        lol = bs.find_all("script")[-5].text[21:]
        lol = json.loads(lol[:-1])
        user = lol["entry_data"]["ProfilePage"][0]["user"]

        #pics = user["media"]["nodes"]
        user["_id"] = user["id"]
        del user["id"]
        user["followers"] = user["followed_by"]["count"]
        user["following"] = user["follows"]["count"]
        user["picture-count"] = user["media"]["count"]
        del user["media"]
        del user["follows"]
        del user["followed_by"]
        return user

while ssdb.qsize("ig-queue"):
    # pop off queue
    urls = ssdb.qpop("ig-queue",1000)

    # write to file
    f = open("links.txt", "w")
    f.write("\n".join(map(lambda x: str(x), urls)))
    f.close()

    try:
        bashCommand = "cat links.txt | parallel --gnu -j20 wget -E"
        output = subprocess.check_output(['bash','-c', bashCommand])
    except Exception as e:
        print e

    # parse file
Beispiel #5
0
class QueueSSDB(QueueBase.QueueBase):
    """
    base class , only provide interface for sub class to implement
    """
    def __init__(self, name, host='localhost', port=8888, **kwargs):
        QueueBase.QueueBase.__init__(self, name, host, port)
        self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=self.host, port=self.port))


    #queue
    @QueueBase.catch
    def put(self, value, *args, **kwargs):
        """
        put an  item in the back of a queue
        :param value:
        :param args:
        :param kwargs:
        :return:
        """
        return self.__conn.qpush_back(self.name,
                                      json.dumps(value, ensure_ascii=False).encode('utf-8') if isinstance(value, dict) or isinstance(value, list) else value)

    def save(self, value, *args, **kwargs):
        """
        put an  item in the back of a queue
        :param value:
        :param args:
        :param kwargs:
        :return:
        """
        return self.__conn.qpush_back(self.name,
                                      json.dumps(value, ensure_ascii = False).encode('utf-8') if isinstance(value,
                                                                                                            dict) or isinstance(
                                          value, list) else value)

    @QueueBase.catch
    def get(self, *args, **kwargs):
        """
        get element from the from of queue
        :param args:
        :param kwargs:
        :return:
        """
        value = self.__conn.qpop_front(self.name)
        return value[0] if value else value

    @QueueBase.catch
    def getMore(self, *args, **kwargs):
        """
        get element from the from of queue
        :param args:
        :param kwargs:
        :return:
        """
        value = self.__conn.qpop_front(self.name, **kwargs)
        return value

    @QueueBase.catch
    def size(self, *args, **kwargs):
        return self.__conn.qsize(self.name)

    @QueueBase.catch
    def changeTable(self, name):
        """
        change the queue name to operate
        :param name:
        :return:
        """
        self.name = name

    @QueueBase.catch
    def select_queue(self, name):
        """
        change the queue name to operate
        :param name:
        :return:
        """
        self.name = name

    @QueueBase.catch
    def qclaerQueue(self):
        return self.__conn.qclear(self.name)


    #KV
    @QueueBase.catch
    def keySet(self,key,value):
        """
        Set the value at key ``name`` to ``value`` .
        :param key:
        :param value:
        :return:
        """
        value = json.dumps(value, ensure_ascii = False).encode('utf-8') if isinstance(value,dict) or isinstance(value, list) else value
        return self.__conn.set(key,value)

    @QueueBase.catch
    def keySetx(self,name, value, ttl=-1):
        """
        Set the value of key ``name`` to ``value`` that expires in ``ttl``
        seconds. ``ttl`` can be represented by an integer or a Python
        timedelta object.
        :param name:
        :param value:
        :param ttl:
        :return:
        """
        return self.__conn.setx(name,value,ttl=ttl)

    @QueueBase.catch
    def keyTtl(self,key):
        """
        Returns the number of seconds until the key ``name`` will expire.
        :return:
        """
        self.__conn.ttl(key)

    @QueueBase.catch
    def keyGet(self,key):
        """
        Return the value at key ``name``, or ``None`` if the key doesn't exist
        :param key:
        :return:
        """
        return self.__conn.get(key)

    @QueueBase.catch
    def keyDel(self,key):
        """
        Delete the key specified by ``name`` .
        :param key:
        :return:
        """
        return self.__conn.delete(key)

    @QueueBase.catch
    def keyKeys(self,key_start='',key_end=''):
        """
        Return a list of the top ``limit`` keys between ``name_start`` and
        ``name_end``
        :param key_start:
        :param key_end:
        :return:
        """
        return self.__conn.keys(name_start=key_start,name_end=key_end,limit=100000)
    @QueueBase.catch
    def keyexists(self,key):
        """
        :param key:
        :return:
        """
        return self.__conn.exists(key)
    #SET
    @QueueBase.catch
    def zsetSet(self,field,score = 1):
        if field:
            if isinstance(field, dict) or isinstance(field, list):
                field = json.dumps(field)
            field = field if len(field) < 100 else field[:100]
        return self.__conn.zset(self.name, field, score)
    @QueueBase.catch
    def zgetSet(self,key):
        return self.__conn.zget(self.name,key)
    @QueueBase.catch
    def zexistsSet(self,name,field):
        return self.__conn.zexists(name,field)
    @QueueBase.catch
    def zkeysSet(self):
        return self.__conn.zkeys(self.name,'','','',limit=100000000)
    @QueueBase.catch
    def zdelSet(self,key):
        return self.__conn.zdel(self.name,key)
    @QueueBase.catch
    def multi_zgetSet(self,*keys):
        return self.__conn.multi_zget(self.name,*keys)
    #Hash
    @QueueBase.catch
    def hgetallHash(self,key):
        return self.__conn.hgetall(key)
    @QueueBase.catch
    def hincrHash(self,name,key):
        return self.__conn.hincr(name,key,amount=1)
    @QueueBase.catch
    def multi_hsetHash(self,name,**mapping):
        return self.__conn.multi_hset(name, **mapping)

    @QueueBase.catch
    def hlistHash(self,start,end):
        return self.__conn.hlist(start, end, limit =  10000000)
    @QueueBase.catch
    def hclearHash(self,key):
        return self.__conn.hclear(key)

    @QueueBase.catch
    def hset(self, key,value):
        return self.__conn.hset(self.name, key,json.dumps(value, ensure_ascii = False).encode('utf-8')
                                if isinstance(value,dict) or isinstance(value, list) else value)

    @QueueBase.catch
    def hsize(self):
        return self.__conn.hsize(self.name)

    @QueueBase.catch
    def hget(self, key = None):
        if key:
            return self.__conn.hget(self.name, key)
        else:
            if self.__conn.hsize(self.name) > 0:
                keys=self.__conn.hkeys(self.name,"", "",limit=1)
                if keys:
                    key=keys[0]
                    v=self.__conn.hget(self.name, key)
                    self.__conn.hdel(self.name, key)
                    return v