def __init__(self, *args, thread_pool_size=1, **kwargs):
#super(SiteThreadChecker, self).__init__(*args, **kwargs)
SiteChecker.__init__(self, *args, output_buff_size=thread_pool_size*50, **kwargs)
self.max_thread = thread_pool_size
LinkChecker.max_http_connection = self.max_thread
LinkChecker.max_pool_size = self.max_thread
#self.pool = multiprocessing.Pool(processes=self.max_thread, maxtasksperchild=1)
self.pool = ThreadPool(processes=self.max_thread)
self._set_task_control_max(self.max_thread)
def __init__(self, *args, thread_pool_size=1, **kwargs):
#super(SiteThreadChecker, self).__init__(*args, **kwargs)
SiteChecker.__init__(self,
*args,
output_buff_size=thread_pool_size * 50,
**kwargs)
self.max_thread = thread_pool_size
LinkChecker.max_http_connection = self.max_thread
LinkChecker.max_pool_size = self.max_thread
#self.pool = multiprocessing.Pool(processes=self.max_thread, maxtasksperchild=1)
self.pool = ThreadPool(processes=self.max_thread)
self._set_task_control_max(self.max_thread)
def get_input_parameter(full_link: str, max_page:int, max_level: int, output_queue, pool_size: int):
temp = SiteChecker.get_input_parameter_base(full_link, max_page, max_level, output_queue)
temp.update({SiteThreadChecker.pool_size_key: pool_size})
return temp
def get_input_parameter(full_link: str, max_page: int, max_level: int,
output_queue, pool_size: int):
temp = SiteChecker.get_input_parameter_base(full_link, max_page,
max_level, output_queue)
temp.update({SiteThreadChecker.pool_size_key: pool_size})
return temp
