def __init__(self, *args, thread_pool_size=1, **kwargs): #super(SiteThreadChecker, self).__init__(*args, **kwargs) SiteChecker.__init__(self, *args, output_buff_size=thread_pool_size*50, **kwargs) self.max_thread = thread_pool_size LinkChecker.max_http_connection = self.max_thread LinkChecker.max_pool_size = self.max_thread #self.pool = multiprocessing.Pool(processes=self.max_thread, maxtasksperchild=1) self.pool = ThreadPool(processes=self.max_thread) self._set_task_control_max(self.max_thread)
def __init__(self, *args, thread_pool_size=1, **kwargs): #super(SiteThreadChecker, self).__init__(*args, **kwargs) SiteChecker.__init__(self, *args, output_buff_size=thread_pool_size * 50, **kwargs) self.max_thread = thread_pool_size LinkChecker.max_http_connection = self.max_thread LinkChecker.max_pool_size = self.max_thread #self.pool = multiprocessing.Pool(processes=self.max_thread, maxtasksperchild=1) self.pool = ThreadPool(processes=self.max_thread) self._set_task_control_max(self.max_thread)
def get_input_parameter(full_link: str, max_page:int, max_level: int, output_queue, pool_size: int): temp = SiteChecker.get_input_parameter_base(full_link, max_page, max_level, output_queue) temp.update({SiteThreadChecker.pool_size_key: pool_size}) return temp
def get_input_parameter(full_link: str, max_page: int, max_level: int, output_queue, pool_size: int): temp = SiteChecker.get_input_parameter_base(full_link, max_page, max_level, output_queue) temp.update({SiteThreadChecker.pool_size_key: pool_size}) return temp