def __init__(self, stop_event: Event, input_queue: Queue=None, output_queue: Queue=None,
max_worker: int=10, dir_path="", is_debug=False, **kwargs):
self._is_debug = is_debug
FeedbackInterface.__init__(self, **kwargs)
ExternalTempInterface.__init__(self)
# do not use predefined queue here
# self._input_q = input_queue
# self._output_q = output_queue
self._stop_event = stop_event
self._internal_stop_event = Event()
self._max_worker = max_worker
self._job_done = 0
self._job_done_shadow = 0
self._job_done_lock = RLock()
self._input_period = 0.0001 # time to sample data into the buffer
self._max_sample_results = 100000
self._min_sampling_duration = 0.0001
self._sample_batch_size = 5000
self._sample_batch_timeout = 60
if is_debug:
self._min_buff_delete_threshold = 10000 # default is 100000
else:
self._min_buff_delete_threshold = 100000
self._speed_penalty_count = 0
self._finished = False
manager, self._output_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Output)
self._db_buffer = ExternalTempDataDiskBuffer("whois_check.db", self, self._internal_stop_event, buf_size=self._max_worker*50,
terminate_callback=WhoisChecker.terminate_callback, dir_path=dir_path)
self._populate_with_state() # FeedbackInterface
if not is_debug:
log_period = 120
else:
log_period = 10
self._progress_logger = ProgressLogger(log_period, self, self._internal_stop_event)
def __init__(self,
stop_event: Event,
input_queue: Queue = None,
output_queue: Queue = None,
max_worker: int = 10,
dir_path="",
is_debug=False,
**kwargs):
self._is_debug = is_debug
FeedbackInterface.__init__(self, **kwargs)
ExternalTempInterface.__init__(self)
# do not use predefined queue here
# self._input_q = input_queue
# self._output_q = output_queue
self._stop_event = stop_event
self._internal_stop_event = Event()
self._max_worker = max_worker
self._job_done = 0
self._job_done_shadow = 0
self._job_done_lock = RLock()
self._input_period = 0.0001 # time to sample data into the buffer
self._max_sample_results = 100000
self._min_sampling_duration = 0.0001
self._sample_batch_size = 5000
self._sample_batch_timeout = 60
if is_debug:
self._min_buff_delete_threshold = 10000 # default is 100000
else:
self._min_buff_delete_threshold = 100000
self._speed_penalty_count = 0
self._finished = False
manager, self._output_q = get_queue_client(
QueueManager.MachineSettingCrawler,
QueueManager.Method_Whois_Output)
self._db_buffer = ExternalTempDataDiskBuffer(
"whois_check.db",
self,
self._internal_stop_event,
buf_size=self._max_worker * 50,
terminate_callback=WhoisChecker.terminate_callback,
dir_path=dir_path)
self._populate_with_state() # FeedbackInterface
if not is_debug:
log_period = 120
else:
log_period = 10
self._progress_logger = ProgressLogger(log_period, self,
self._internal_stop_event)
def __init__(self, db_ref: str, db_dir: str, input_queue: Queue, output_queue: Queue, stop_event: Event,
matrix: CrawlMatrix, accounts: list, force_mode=False, force_mode_offset=0, force_mode_total=0, **kwargs):
FeedbackInterface.__init__(self, **kwargs)
self._stop_event = stop_event
self._matrix = matrix
self._db_ref = db_ref
self._input_queue = input_queue
self._output_queue = output_queue
self._pool_input = Queue()
self._pool = FilterPool(self._pool_input, self._output_queue, self._queue_lock, self._stop_event, self._matrix,
accounts=accounts)
self._db_buffer = ExternalTempDataDiskBuffer(self._db_ref, self, self._stop_event, dir_path=db_dir,
buf_size=2500, output_f=5000) # control how data flow speed,
# it can keep input:output ratio = 1:1 at max 10 milion data row per hour
#FeedbackInterface.__init__(self, **kwargs)
ExternalTempInterface.__init__(self)
self._populate_with_state()
if force_mode:
new_state = _FilterState(progress=force_mode_offset, all_data=force_mode_total)
self.populate_with_state(new_state)
def __init__(self,
db_ref: str,
db_dir: str,
input_queue: Queue,
output_queue: Queue,
stop_event: Event,
matrix: CrawlMatrix,
accounts: list,
force_mode=False,
force_mode_offset=0,
force_mode_total=0,
**kwargs):
FeedbackInterface.__init__(self, **kwargs)
self._stop_event = stop_event
self._matrix = matrix
self._db_ref = db_ref
self._input_queue = input_queue
self._output_queue = output_queue
self._pool_input = Queue()
self._pool = FilterPool(self._pool_input,
self._output_queue,
self._queue_lock,
self._stop_event,
self._matrix,
accounts=accounts)
self._db_buffer = ExternalTempDataDiskBuffer(
self._db_ref,
self,
self._stop_event,
dir_path=db_dir,
buf_size=2500,
output_f=5000) # control how data flow speed,
# it can keep input:output ratio = 1:1 at max 10 milion data row per hour
#FeedbackInterface.__init__(self, **kwargs)
ExternalTempInterface.__init__(self)
self._populate_with_state()
if force_mode:
new_state = _FilterState(progress=force_mode_offset,
all_data=force_mode_total)
self.populate_with_state(new_state)
def __init__(self, full_link: str="", data_source: SiteTempDataSrcInterface=None,
controller: SiteCheckerController=None,
max_level=10, max_page=1000, delegate=None, output_buff_size=2000,
output_queue=None, output_all_external=False, result_delegate=None,
memory_control_terminate_event=None, check_robot_text=True,
**kwargs):
"""
:param full_link: The full link of a domain, e.g: https://www.google.co.uk
:param domain: domain to crawl
:param max_level: stop crawling if it reaches this level
:param max_page: maximum pages to check within a site, also stop crawling
:param delegate: if this is not None, then it will send the latest result of external domain of ResponseCode==404 or 999
:param result_delegate: send site_info upon finish
:param memory_control_terminate_event: if this is not None and being set, it will be able to terminate an external memory controlled process.
:return:
"""
FeedbackInterface.__init__(self, **kwargs)
#super(SiteChecker, self).__init__(**kwargs)
if full_link is None or len(full_link) == 0:
raise ValueError()
original_path = ""
try:
paras = urlsplit(full_link)
self.scheme, self.domain, original_path = paras[0], paras[1], paras[2]
except:
pass
domain_data = LinkChecker.get_root_domain(full_link, False)
self.root_domain = domain_data[1]
self.sub_domain = domain_data[4]
self.domain_suffix = domain_data[5]
self.sub_domain_no_local = self.sub_domain.strip(self.domain_suffix)
if self.scheme == "":
self.scheme = "http"
if self.domain == "":
self.domain = self.root_domain
self.orginal_link = full_link
self.domain_link = LinkChecker.get_valid_link(self.root_domain, full_link, self.scheme)
self.max_level = max_level
self.max_page = max_page
self.page_count = 0 # keep track page done
self._page_count_shadow = 0 # track previous count
self._all_page_count_shadow = 0 #track previous count in datasource
self.internal_page_count = 0
self.internal_page_last_count = 0
self.page_allocated = 0
self.current_level = 0 # if this = 0, it is root domain/home_page
self._stop_event = Event()
valid_file_name = SiteTempDataSrcInterface.get_valid_file_name(self.domain_link)
self._external_db_buffer = ExternalTempDataDiskBuffer(valid_file_name+".ext.db", self,
stop_event=self._stop_event,
buf_size=int(output_buff_size/2),
dir_path=get_db_buffer_default_dir(),
convert_output=False)
self._external_db_buffer.append_to_buffer([(self.root_domain, ResponseCode.DNSError),], convert_tuple=False)
self._memory_control_terminate_event = memory_control_terminate_event
self.task_control_lock = threading.RLock()
if data_source is None:
#self.data_source = SiteTempDataDisk(self.root_domain, ref_obj=self)
self.data_source = SiteTempDataDiskWithBuff(ref=self.domain_link, output_buff_size=output_buff_size, ref_obj=self)
else:
self.data_source = data_source # a list of OnSiteLink
self.delegate = delegate
if LinkChecker.might_be_link_html_page(original_path):
self.data_source.append(OnSiteLink(self.domain_link, response_code=ResponseCode.LinkOK, link_level=1)) # add the root domain as a starting point
self.data_source.append(OnSiteLink(self.scheme + "://www."+self.sub_domain, ResponseCode.LinkOK, link_level=1))
self.data_source.append(OnSiteLink(self.scheme + "://" + self.domain, ResponseCode.LinkOK, link_level=1))
self.cache_list = [] # internal page cache
self.page_need_look_up_temp = 0
self.cache_list.append(self.domain_link)
if "www." not in self.sub_domain:
self.cache_list.append(self.scheme + "://www."+self.sub_domain)
self.cache_list.append(self.scheme + "://" + self.domain)
self.page_need_look_up = self.data_source.count_all()
self.cache_size = 500 # create a small cache list to avoid going to check link in file system with lots of read and write
self._double_check_cache_lock = threading.RLock()
self._double_check_cache = deque(maxlen=self.cache_size)
self.external_cache_list = []
self.external_cache_size = 500 # cache that hold external sites
self.external_links_checked = 0
self.add_internal_page_OK_only = True
self.output_queue = output_queue
self.output_all_external = output_all_external
self.controller = controller
self.result_delegate = result_delegate
self.page_count_lock = threading.RLock()
self.internal_page_count_lock = threading.RLock()
self.level_lock = threading.RLock()
self.page_look_up_lock = threading.RLock()
self.external_link_check_lock = threading.RLock()
self._finihsed = False
self.task_control_max = 1
self.agent = "VegeBot (we follow your robots.txt settings before crawling, you can slow down the bot by change the Crawl-Delay parameter in the settings." \
"if you have an enquiry, please email to: [email protected])"
self.agent_from = "*****@*****.**"
if check_robot_text:
self.robot_agent = LinkChecker.get_robot_agent(self.sub_domain, protocol=self.scheme)
else:
self.robot_agent = None
self.site_crawl_delay = 0.60
if isinstance(self.robot_agent, Rules):
delay_temp = self.robot_agent.delay(self.agent)
if delay_temp is not None and delay_temp != self.site_crawl_delay:
self.site_crawl_delay = delay_temp
self.task_control_counter = 1
self._speed_penalty_count = 0
self._speed_penalty_threshold = 10
self._progress_logging_speed = 120
self._output_period = 120
self._output_batch_size = 100
self._death_wish_sent = False
SiteChecker._is_lxml_parser_exist()
self._output_thread = None
self._output_queue = None
self.progress_logger = ProgressLogger(self._progress_logging_speed, self, self._stop_event)
self._status = "Start"
self._populate_with_state() # restore laste known state
