예제 #1
0
    def cateGoodsControl(self, n):
        # 恢复爬取状态
        tp = ThreadPool(n, max_task_num=1)
        while True:
            if not tp.q.empty():
                continue

            state = self._loadState()
            leave_list = list(range(183))
            leave_list.reverse()
            if state:
                goods_down = state.get('goods_down', [])
                leave_list = list(set(leave_list) - set(goods_down))

            if not leave_list:
                print("All Things is Done!!!")
                break

            j = 0
            for i in leave_list:
                print(i)
                tp.put(self.cateGoods, (i, ))
                if j == 0:
                    time.sleep(60)
                    j = 1
            time.sleep(3)
예제 #2
0
    def cateGoodsControl(self, n):
        # 恢复爬取状态
        tp = ThreadPool(n)
        while True:
            state = self._loadState()
            leave_list = list(range(183))
            if state:
                goods_down = state.get('goods_down', [])
                leave_list = list(set(leave_list) - set(goods_down))

            j = 0
            for i in leave_list:
                print(i)
                tp.put(self.cateGoods, (i, ))
                if j == 0:
                    time.sleep(60)
                    j = 1