コード例 #1
0
ファイル: start.py プロジェクト: AlexLyj/aminer-spider
	def _maintainThreadPool(self, reload_all_thread):
		'''
		Maintain ThreadPool, detect and restart, and set running threads on the fly.
		'''
		num_persont_alive = 0
		num_pubt_alive = 0

#		if reload_all_thread: # kill all thread first.
		if False:#we don't want reload fucntion until fix the bug : pub thread idle  
			for idx_pub_t in range(0, self.settings.max_pub_thread):
				t = None
				if len(self.pub_thread_pool) <= idx_pub_t:
					self.pub_thread_pool.append(t)
				else:
					t = self.pub_thread_pool[idx_pub_t]
				if t is not None:
					t.ask_to_stop = True
			self.pub_thread_pool = []

		#fill thread pool
		while len(self.person_thread_pool)<self.settings.max_person_thread:
			self.person_thread_pool.append(None)

		while len(self.pub_thread_pool)<self.settings.max_pub_thread:
			self.pub_thread_pool.append(None)

		# check and start all unstarted threads.
		idx_person_t = 0
		for idx_person_t in range(0, self.settings.max_person_thread):
			t = self.person_thread_pool[idx_person_t]

			if t is None: # if is None(new add) or dead.
				if self.running:
					t = PersonProcessThread(self)
					t.name = 'person-thread-' + str(idx_person_t)
					self.person_thread_pool[idx_person_t] = t
					t.start()
					with self.busy_semaphore_lock:
						self.threadChildren += 1
						num_persont_alive += 1
			elif not t.is_alive() or not t.check_idle():
				if self.running:
					killedname = t.name
					t.stop()
					print "$mgr/thread:> kill thread %s" % killedname
					t = PersonProcessThread(self)
					t.name = 'person-thread-' + str(idx_person_t)
					self.person_thread_pool[idx_person_t] = t
					t.start()
					with self.busy_semaphore_lock:
						self.threadChildren += 1
						num_persont_alive += 1
			else:
				num_persont_alive += 1

		# check and start all unstarted threads.
		idx_pub_t = 0
		for idx_pub_t in range(0, self.settings.max_pub_thread):
			t = self.pub_thread_pool[idx_pub_t]
			
			if t is None:
				if self.running:
					t = PubProcessThread(self)
					t.name = 'pub-thread-' + str(idx_pub_t)
					self.pub_thread_pool[idx_pub_t] = t
					t.start()
					with self.busy_semaphore_lock:
						self.threadChildren += 1
						num_pubt_alive += 1
			elif not t.is_alive() or not t.check_idle():
				if self.running:
					killedname = t.name
					t.stop()
					print "$mgr/thread:> kill thread %s" % killedname
					t = PubProcessThread(self)
					t.name = 'pub-thread-' + str(idx_pub_t)
					self.pub_thread_pool[idx_pub_t] = t
					t.start()
					with self.busy_semaphore_lock:
						self.threadChildren += 1
						num_pubt_alive += 1
			else:
				num_pubt_alive += 1

		return (num_persont_alive, num_pubt_alive)
コード例 #2
0
    def _maintainThreadPool(self, reload_all_thread):
        '''
                Maintain ThreadPool, detect and restart, and set running threads on the fly.
		'''
        # Collect Information.
        num_persont_alive = 0
        num_pubt_alive = 0

        if reload_all_thread:  # kill all thread first.
            for idx_pub_t in range(0, self.max_pub_thread):
                t = None
                if len(self.pub_thread_pool) <= idx_pub_t:
                    self.pub_thread_pool.append(t)
                else:
                    t = self.pub_thread_pool[idx_pub_t]
                if t is not None:
                    t.ask_to_stop = True
            self.pub_thread_pool = []

        # check and start all unstarted threads.
        idx_person_t = 0
        for idx_person_t in range(0, self.max_person_thread):
            t = None
            if len(self.person_thread_pool) <= idx_person_t:
                self.person_thread_pool.append(
                    t)  # if len less than max size, increase with None.
            else:
                t = self.person_thread_pool[idx_person_t]

            if t is None or not t.is_alive():  # if is None(new add) or dead.
                if self.running:
                    t = PersonProcessThread(self)
                    t.name = 'person-thread-' + str(idx_person_t)
                    self.person_thread_pool[idx_person_t] = t
                    t.start()
                    num_persont_alive += 1
            else:
                num_persont_alive += 1

        # kill threads if needed.
        for i in range(idx_person_t,
                       len(self.person_thread_pool) - 1):  #@UnusedVariable
            t = self.person_thread_pool.pop(idx_person_t)
            t.stop()
            print "$mgr/thread:> kill thread %s" % t.name

        # check and start all unstarted threads.
        idx_pub_t = 0
        for idx_pub_t in range(0, self.max_pub_thread):
            t = None
            if len(self.pub_thread_pool) <= idx_pub_t:
                self.pub_thread_pool.append(t)
            else:
                t = self.pub_thread_pool[idx_pub_t]

            if t is None or not t.is_alive():
                if self.running:
                    t = PubProcessThread(self)
                    t.name = 'pub-thread-' + str(idx_pub_t)
                    self.pub_thread_pool[idx_pub_t] = t
                    t.start()
                    num_pubt_alive += 1
            else:
                num_pubt_alive += 1

        # kill threads if needed.
        for i in range(idx_pub_t,
                       len(self.pub_thread_pool) - 1):  #@UnusedVariable
            t = self.pub_thread_pool.pop(idx_pub_t)
            t.stop()
            print "$mgr/thread:> kill thread %s" % t.name

        return (num_persont_alive, num_pubt_alive)
コード例 #3
0
ファイル: start_cw.py プロジェクト: Rygbee/aminer-spider
    def _maintainThreadPool(self, reload_all_thread):
        """
                Maintain ThreadPool, detect and restart, and set running threads on the fly.
		"""
        # Collect Information.
        num_persont_alive = 0
        num_pubt_alive = 0

        if reload_all_thread:  # kill all thread first.
            for idx_pub_t in range(0, self.max_pub_thread):
                t = None
                if len(self.pub_thread_pool) <= idx_pub_t:
                    self.pub_thread_pool.append(t)
                else:
                    t = self.pub_thread_pool[idx_pub_t]
                if t is not None:
                    t.ask_to_stop = True
            self.pub_thread_pool = []

            # check and start all unstarted threads.
        idx_person_t = 0
        for idx_person_t in range(0, self.max_person_thread):
            t = None
            if len(self.person_thread_pool) <= idx_person_t:
                self.person_thread_pool.append(t)  # if len less than max size, increase with None.
            else:
                t = self.person_thread_pool[idx_person_t]

            if t is None or not t.is_alive():  # if is None(new add) or dead.
                if self.running:
                    t = PersonProcessThread(self)
                    t.name = "person-thread-" + str(idx_person_t)
                    self.person_thread_pool[idx_person_t] = t
                    t.start()
                    num_persont_alive += 1
            else:
                num_persont_alive += 1

                # kill threads if needed.
        for i in range(idx_person_t, len(self.person_thread_pool) - 1):  # @UnusedVariable
            t = self.person_thread_pool.pop(idx_person_t)
            t.stop()
            print "$mgr/thread:> kill thread %s" % t.name

            # check and start all unstarted threads.
        idx_pub_t = 0
        for idx_pub_t in range(0, self.max_pub_thread):
            t = None
            if len(self.pub_thread_pool) <= idx_pub_t:
                self.pub_thread_pool.append(t)
            else:
                t = self.pub_thread_pool[idx_pub_t]

            if t is None or not t.is_alive():
                if self.running:
                    t = PubProcessThread(self)
                    t.name = "pub-thread-" + str(idx_pub_t)
                    self.pub_thread_pool[idx_pub_t] = t
                    t.start()
                    num_pubt_alive += 1
            else:
                num_pubt_alive += 1

                # kill threads if needed.
        for i in range(idx_pub_t, len(self.pub_thread_pool) - 1):  # @UnusedVariable
            t = self.pub_thread_pool.pop(idx_pub_t)
            t.stop()
            print "$mgr/thread:> kill thread %s" % t.name

        return (num_persont_alive, num_pubt_alive)
コード例 #4
0
ファイル: start.py プロジェクト: yinonbaron/aminer-spider
    def _maintainThreadPool(self, reload_all_thread):
        '''
		Maintain ThreadPool, detect and restart, and set running threads on the fly.
		'''
        num_persont_alive = 0
        num_pubt_alive = 0

        #		if reload_all_thread: # kill all thread first.
        if False:  #we don't want reload fucntion until fix the bug : pub thread idle
            for idx_pub_t in range(0, self.settings.max_pub_thread):
                t = None
                if len(self.pub_thread_pool) <= idx_pub_t:
                    self.pub_thread_pool.append(t)
                else:
                    t = self.pub_thread_pool[idx_pub_t]
                if t is not None:
                    t.ask_to_stop = True
            self.pub_thread_pool = []

        #fill thread pool
        while len(self.person_thread_pool) < self.settings.max_person_thread:
            self.person_thread_pool.append(None)

        while len(self.pub_thread_pool) < self.settings.max_pub_thread:
            self.pub_thread_pool.append(None)

        # check and start all unstarted threads.
        idx_person_t = 0
        for idx_person_t in range(0, self.settings.max_person_thread):
            t = self.person_thread_pool[idx_person_t]

            if t is None:  # if is None(new add) or dead.
                if self.running:
                    t = PersonProcessThread(self)
                    t.name = 'person-thread-' + str(idx_person_t)
                    self.person_thread_pool[idx_person_t] = t
                    t.start()
                    with self.busy_semaphore_lock:
                        self.threadChildren += 1
                        num_persont_alive += 1
            elif not t.is_alive() or not t.check_idle():
                if self.running:
                    killedname = t.name
                    t.stop()
                    print "$mgr/thread:> kill thread %s" % killedname
                    t = PersonProcessThread(self)
                    t.name = 'person-thread-' + str(idx_person_t)
                    self.person_thread_pool[idx_person_t] = t
                    t.start()
                    with self.busy_semaphore_lock:
                        self.threadChildren += 1
                        num_persont_alive += 1
            else:
                num_persont_alive += 1

        # check and start all unstarted threads.
        idx_pub_t = 0
        for idx_pub_t in range(0, self.settings.max_pub_thread):
            t = self.pub_thread_pool[idx_pub_t]

            if t is None:
                if self.running:
                    t = PubProcessThread(self)
                    t.name = 'pub-thread-' + str(idx_pub_t)
                    self.pub_thread_pool[idx_pub_t] = t
                    t.start()
                    with self.busy_semaphore_lock:
                        self.threadChildren += 1
                        num_pubt_alive += 1
            elif not t.is_alive() or not t.check_idle():
                if self.running:
                    killedname = t.name
                    t.stop()
                    print "$mgr/thread:> kill thread %s" % killedname
                    t = PubProcessThread(self)
                    t.name = 'pub-thread-' + str(idx_pub_t)
                    self.pub_thread_pool[idx_pub_t] = t
                    t.start()
                    with self.busy_semaphore_lock:
                        self.threadChildren += 1
                        num_pubt_alive += 1
            else:
                num_pubt_alive += 1

        return (num_persont_alive, num_pubt_alive)