def _normalize_links(self, url, link_infos):
links = []
for link, _ in link_infos:
link = url_analyser.normalize_url(link, url)
if link is not None and link != url:
links.append(link)
return links
def _process(self, message):
# normalize url
url = url_analyser.normalize_url(message["url"])
if url is None:
logging.error("invalid url for crawl", url = message["url"])
return {"status" : -1}
message["url"] = url
#fill optional fields
url_info = misc.clone_dict(message, fields = ["url", "source", "root_url", "parent_url", "crawl_priority", "crawl_depth"])
self._assign_url_info_defaults(url_info)
if url_info["root_url"] is None:
url_info["root_url"] = url
#deterimine crawl priority/depth
is_valid, url_info["crawl_priority"], url_info["crawl_depth"] = crawl_priority_and_depth_evaluator.evaluate(url, url_info["source"], url_info)
if not is_valid:
return {"status" : -1}
# stores to urlRepository table
url_info["page_last_modified"] = None
url_info["crawl_status"] = "crawling"
url_info["last_crawled"] = None
url_info["original_url"] = None
# all urls is static now
url_info["crawl_type"] = "static"
# TODO add to crawler db, this should not be done here
# some project do not need to store url info into database
# should use middleware for these kind of actions
#success, promoted = crawlerdb.add_url_info(url, url_info, True)
if message["source"] != "redirected":
# notify crawler
message_type, crawler_message = CrawlerUtils.build_crawler_request_msg(url, url_info)
handler.HandlerRepository.process(message_type, crawler_message)
return {"status" : 1}
