예제 #1
0
	def run(self, session_id):
		"""
			Run the main function ...
		"""

		self.thread_count = self.config_result["content_thread"]

		# debug
		#print "Thread count to run mount %s"% self.thread_count

		pool = ThreadPool(int(self.thread_count))
		if session_id == 0:
			try:
				read_file = open(self.share_session, "r").read().splitlines()
			except Exception, err_mess:
				print err_mess
				sys.exit(1)	
			
			for line in read_file:	
				pool.add_task(self.mount_sharing, line)

			pool.wait_completion()

			self.write_session_id(str(session_id))
			session_id = session_id + 1
예제 #2
0
    def cateGoodsControl(self, n):
        # 恢复爬取状态
        tp = ThreadPool(n, max_task_num=1)
        while True:
            if not tp.q.empty():
                continue

            state = self._loadState()
            leave_list = list(range(183))
            leave_list.reverse()
            if state:
                goods_down = state.get('goods_down', [])
                leave_list = list(set(leave_list) - set(goods_down))

            if not leave_list:
                print("All Things is Done!!!")
                break

            j = 0
            for i in leave_list:
                print(i)
                tp.put(self.cateGoods, (i, ))
                if j == 0:
                    time.sleep(60)
                    j = 1
            time.sleep(3)
예제 #3
0
    def run(self):
        pool = ThreadPool(10)
        while self.running:
            requests = makeRequests(some_callable, list_of_args)
            [pool.putRequest(req) for req in requests]
            pool.wait()

            sleep(1)
            print "ActionHandlerThread"
예제 #4
0
    def cateGoodsControl(self, n):
        # 恢复爬取状态
        tp = ThreadPool(n)
        while True:
            state = self._loadState()
            leave_list = list(range(183))
            if state:
                goods_down = state.get('goods_down', [])
                leave_list = list(set(leave_list) - set(goods_down))

            j = 0
            for i in leave_list:
                print(i)
                tp.put(self.cateGoods, (i, ))
                if j == 0:
                    time.sleep(60)
                    j = 1
예제 #5
0
def process_user(uid):
    auth_profile = AuthProfile.objects.filter(dropbox_userid=uid)[0]
    token = auth_profile.dropbox_access_token
    user = auth_profile.user
    logger.info("START process user '%s'" % user)
    logger.debug("Process change notfication for user: %s" % user.username)
    cursor = cache.get("cursor-%s" % uid)

    client = Connection(token)

    has_more = True
    while has_more:
        result = client.delta(cursor)

        pool = ThreadPool(1)
        for path, metadata in result['entries']:
            logger.info("Add task for %s to pool" % path)
            pool.add_task(process_file, path, metadata, client, user)
            logger.debug("Waiting for completion ... %s" % path)
        pool.wait_completion()
        logger.debug("Tasks completed.")

        # Update cursor
        cursor = result['cursor']
        cursor = cache.set("cursor-%s" % uid, cursor)

        # Repeat only if there's more to do
        has_more = result['has_more']

    logger.info("END process user '%s'" % user)
예제 #6
0
    def run(self, session_id):
        """
			Run the main function ...
		"""

        self.thread_count = self.config_result["content_thread"]

        # debug
        if self.verbose > 0:
            print "   [+] Thread count to run mount %s" % self.thread_count

        pool = ThreadPool(int(self.thread_count))
        if session_id == 0:
            try:
                read_file = open(self.share_session, "r").read().splitlines()
            except Exception, err_mess:
                print err_mess
                sys.exit(1)

            for line in read_file:
                pool.add_task(self.mount_sharing, line)

            pool.wait_completion()

            self.write_session_id(str(session_id))
            session_id = session_id + 1
예제 #7
0
            print(ctime(), 'IDSET = ', len(IDSETs[0]), len(IDSETs[1]), len(IDSETs[2]))
            print(ctime(), 'dbQueue = ', db_tp.q.qsize())
            print(ctime(), 'parseQueue = ', parse_tp.q.qsize())
            ins = input()
        except:
            traceback.print_exc()


def loadIdSet():
    IDSET = dbo.getallidlist()
    return IDSET


if __name__ == "__main__":
    dbo = DbOperate()
    IDSET = loadIdSet()
    IDSETs = [set(IDSET.get('goods', ())), set(IDSET.get('opts', ())), set(IDSET.get('top_opts', ()))]
    Dupli = 0
    parse = Parse()
    # 解析线程池
    parse_tp = ThreadPool(2)
    # 数据库操作线程池,
    db_tp = ThreadPool(1)
    threading.Thread(target=Monitor).start()
    baseproxy = AsyncMitmProxy(server_addr=('', 8888), https=True)
    baseproxy.register(RspInterceptor)
    baseproxy.serve_forever()



예제 #8
0
		#print "Thread count to run mount %s"% self.thread_count

		pool = ThreadPool(int(self.thread_count))
		if session_id == 0:
			try:
				read_file = open(self.share_session, "r").read().splitlines()
			except Exception, err_mess:
				print err_mess
				sys.exit(1)	
			
			for line in read_file:	
				pool.add_task(self.mount_sharing, line)

			pool.wait_completion()

			self.write_session_id(str(session_id))
			session_id = session_id + 1
		else:
			# debug 
			#print "Session files will be used ..."
	
			pool = ThreadPool(int(self.thread_count))
			while self.get_session(session_id):
				line = self.get_session(session_id)
				pool.add_task(self.mount_sharing, line)

				self.write_session_id(str(session_id))
				session_id = int(session_id) + 1
			
			pool.wait_completion()
예제 #9
0
        pool = ThreadPool(int(self.thread_count))
        if session_id == 0:
            try:
                read_file = open(self.share_session, "r").read().splitlines()
            except Exception, err_mess:
                print err_mess
                sys.exit(1)

            for line in read_file:
                pool.add_task(self.mount_sharing, line)

            pool.wait_completion()

            self.write_session_id(str(session_id))
            session_id = session_id + 1
        else:
            # debug
            if self.verbose > 0:
                print "   [+] Session files will be used ..."

            pool = ThreadPool(int(self.thread_count))
            while self.get_session(session_id):
                line = self.get_session(session_id)
                pool.add_task(self.mount_sharing, line)

                self.write_session_id(str(session_id))
                session_id = int(session_id) + 1

            pool.wait_completion()
예제 #10
0
 def __create_pool(self, poolnode):
     """\brief Creates a thread pool
     \param poolnode (\c xml.dom.Node) The thread pool in xml form
     """
     self.__thread_pools[poolnode.attributes['id'].value] = ThreadPool(poolnode,\
                                                                       self.__blockmon)