def testRobot4(self):
#rules = LinkChecker.get_robot_agent("sbnet.se")
rules = LinkChecker.get_robot_agent("realwire.com")
crawl_delay = rules.delay("idiot")
print("delay is:", crawl_delay)
for i in range(1, 1000):
print(rules.allowed("http://api.google.com/search/", agent="idiot"))
def testRobot2(self):
rp = LinkChecker.get_robot_agent("http://pointshound.com/robots.txt")
if rp is not None:
for i in range(1, 1000):
print("count:", i, "can fetch:", rp.can_fetch("*", "http://www.bbc.co.uk/fafdjiaofpadpvhagaarga/news/agqrgfv/y"))
else:
print("domain is not available.")
def testRobot4(self):
#rules = LinkChecker.get_robot_agent("sbnet.se")
rules = LinkChecker.get_robot_agent("realwire.com")
crawl_delay = rules.delay("idiot")
print("delay is:", crawl_delay)
for i in range(1, 1000):
print(rules.allowed("http://api.google.com/search/",
agent="idiot"))
def testRobot5(self):
base_link = "http://pointshound.com"
test_sub_paths = [
"/", "/why", "/about", "/privacy", "/howitworks", "/help",
"/press", "/terms", "/guarantee", "/contact_form", "/something-else"]
rules = LinkChecker.get_robot_agent("pointshound.com", protocol="https")
for item in test_sub_paths:
path = base_link + item
is_allowed = rules.allowed(path, agent="VegeBot Test")
print("sub_path:", item, " is allowed:", is_allowed)
def testRobot2(self):
rp = LinkChecker.get_robot_agent("http://pointshound.com/robots.txt")
if rp is not None:
for i in range(1, 1000):
print(
"count:", i, "can fetch:",
rp.can_fetch(
"*",
"http://www.bbc.co.uk/fafdjiaofpadpvhagaarga/news/agqrgfv/y"
))
else:
print("domain is not available.")
def testRobot5(self):
base_link = "http://pointshound.com"
test_sub_paths = [
"/", "/why", "/about", "/privacy", "/howitworks", "/help",
"/press", "/terms", "/guarantee", "/contact_form",
"/something-else"
]
rules = LinkChecker.get_robot_agent("pointshound.com",
protocol="https")
for item in test_sub_paths:
path = base_link + item
is_allowed = rules.allowed(path, agent="VegeBot Test")
print("sub_path:", item, " is allowed:", is_allowed)
def testGetAgent(self):
root_domain = "halifaxnational.com"
agent = LinkChecker.get_robot_agent(root_domain)
can_fetch = agent.can_fetch("*", "http://halifaxnational.com/somethin")
print(agent,"can fetch:", can_fetch)
def __init__(self, full_link: str="", data_source: SiteTempDataSrcInterface=None,
controller: SiteCheckerController=None,
max_level=10, max_page=1000, delegate=None, output_buff_size=2000,
output_queue=None, output_all_external=False, result_delegate=None,
memory_control_terminate_event=None, check_robot_text=True,
**kwargs):
"""
:param full_link: The full link of a domain, e.g: https://www.google.co.uk
:param domain: domain to crawl
:param max_level: stop crawling if it reaches this level
:param max_page: maximum pages to check within a site, also stop crawling
:param delegate: if this is not None, then it will send the latest result of external domain of ResponseCode==404 or 999
:param result_delegate: send site_info upon finish
:param memory_control_terminate_event: if this is not None and being set, it will be able to terminate an external memory controlled process.
:return:
"""
FeedbackInterface.__init__(self, **kwargs)
#super(SiteChecker, self).__init__(**kwargs)
if full_link is None or len(full_link) == 0:
raise ValueError()
original_path = ""
try:
paras = urlsplit(full_link)
self.scheme, self.domain, original_path = paras[0], paras[1], paras[2]
except:
pass
domain_data = LinkChecker.get_root_domain(full_link, False)
self.root_domain = domain_data[1]
self.sub_domain = domain_data[4]
self.domain_suffix = domain_data[5]
self.sub_domain_no_local = self.sub_domain.strip(self.domain_suffix)
if self.scheme == "":
self.scheme = "http"
if self.domain == "":
self.domain = self.root_domain
self.orginal_link = full_link
self.domain_link = LinkChecker.get_valid_link(self.root_domain, full_link, self.scheme)
self.max_level = max_level
self.max_page = max_page
self.page_count = 0 # keep track page done
self._page_count_shadow = 0 # track previous count
self._all_page_count_shadow = 0 #track previous count in datasource
self.internal_page_count = 0
self.internal_page_last_count = 0
self.page_allocated = 0
self.current_level = 0 # if this = 0, it is root domain/home_page
self._stop_event = Event()
valid_file_name = SiteTempDataSrcInterface.get_valid_file_name(self.domain_link)
self._external_db_buffer = ExternalTempDataDiskBuffer(valid_file_name+".ext.db", self,
stop_event=self._stop_event,
buf_size=int(output_buff_size/2),
dir_path=get_db_buffer_default_dir(),
convert_output=False)
self._external_db_buffer.append_to_buffer([(self.root_domain, ResponseCode.DNSError),], convert_tuple=False)
self._memory_control_terminate_event = memory_control_terminate_event
self.task_control_lock = threading.RLock()
if data_source is None:
#self.data_source = SiteTempDataDisk(self.root_domain, ref_obj=self)
self.data_source = SiteTempDataDiskWithBuff(ref=self.domain_link, output_buff_size=output_buff_size, ref_obj=self)
else:
self.data_source = data_source # a list of OnSiteLink
self.delegate = delegate
if LinkChecker.might_be_link_html_page(original_path):
self.data_source.append(OnSiteLink(self.domain_link, response_code=ResponseCode.LinkOK, link_level=1)) # add the root domain as a starting point
self.data_source.append(OnSiteLink(self.scheme + "://www."+self.sub_domain, ResponseCode.LinkOK, link_level=1))
self.data_source.append(OnSiteLink(self.scheme + "://" + self.domain, ResponseCode.LinkOK, link_level=1))
self.cache_list = [] # internal page cache
self.page_need_look_up_temp = 0
self.cache_list.append(self.domain_link)
if "www." not in self.sub_domain:
self.cache_list.append(self.scheme + "://www."+self.sub_domain)
self.cache_list.append(self.scheme + "://" + self.domain)
self.page_need_look_up = self.data_source.count_all()
self.cache_size = 500 # create a small cache list to avoid going to check link in file system with lots of read and write
self._double_check_cache_lock = threading.RLock()
self._double_check_cache = deque(maxlen=self.cache_size)
self.external_cache_list = []
self.external_cache_size = 500 # cache that hold external sites
self.external_links_checked = 0
self.add_internal_page_OK_only = True
self.output_queue = output_queue
self.output_all_external = output_all_external
self.controller = controller
self.result_delegate = result_delegate
self.page_count_lock = threading.RLock()
self.internal_page_count_lock = threading.RLock()
self.level_lock = threading.RLock()
self.page_look_up_lock = threading.RLock()
self.external_link_check_lock = threading.RLock()
self._finihsed = False
self.task_control_max = 1
self.agent = "VegeBot (we follow your robots.txt settings before crawling, you can slow down the bot by change the Crawl-Delay parameter in the settings." \
"if you have an enquiry, please email to: [email protected])"
self.agent_from = "*****@*****.**"
if check_robot_text:
self.robot_agent = LinkChecker.get_robot_agent(self.sub_domain, protocol=self.scheme)
else:
self.robot_agent = None
self.site_crawl_delay = 0.60
if isinstance(self.robot_agent, Rules):
delay_temp = self.robot_agent.delay(self.agent)
if delay_temp is not None and delay_temp != self.site_crawl_delay:
self.site_crawl_delay = delay_temp
self.task_control_counter = 1
self._speed_penalty_count = 0
self._speed_penalty_threshold = 10
self._progress_logging_speed = 120
self._output_period = 120
self._output_batch_size = 100
self._death_wish_sent = False
SiteChecker._is_lxml_parser_exist()
self._output_thread = None
self._output_queue = None
self.progress_logger = ProgressLogger(self._progress_logging_speed, self, self._stop_event)
self._status = "Start"
self._populate_with_state() # restore laste known state
def testGetAgent(self):
root_domain = "halifaxnational.com"
agent = LinkChecker.get_robot_agent(root_domain)
can_fetch = agent.can_fetch("*", "http://halifaxnational.com/somethin")
print(agent, "can fetch:", can_fetch)
