def main():
"""Start the scanner."""
print_banner()
setup_logger()
args = require_arguments()
options = Options()
options.scope.protocol_must_match = args.protocol_must_match
options.scope.subdomain_must_match = not args.scan_other_subdomains
options.scope.hostname_must_match = not args.scan_other_hostnames
options.scope.tld_must_match = not args.scan_other_tlds
options.scope.max_depth = args.max_depth if args.crawl else 0
options.performance.max_threads = args.max_threads
options.misc.verify_ssl_certificates = not args.ignore_invalid_certificates
options.misc.trusted_certificates = args.trusted_certificates
""" ########################################################## """
""" """
""" Please add your custom options like e.g. authentication, """
""" cookies, headers, proxies or scope options below. """
""" """
""" ########################################################## """
driver = Driver(args, options)
driver.start()
def test_hash_different_encoded_and_decoded_values(self):
"""Ensure encoded and decoded values have a different hash."""
queue = Queue(Options())
queue.add_request(Request("http://example.ltd?val={{aaaa}}"))
queue.add_request(Request("http://example.ltd?val=%7B%7Baaaa%7D%7D"))
self.assertEqual(queue.count_total, 2)
def test_hash_different_query_order(self):
"""Ensure query parameters in different orders are treated as one queue item."""
queue = Queue(Options())
queue.add_request(Request("https://www.example.ltd?b=b&c=c&a=a"))
queue.add_request(Request("https://www.example.ltd?b=b&a=a&c=c"))
queue.add_request(Request("https://www.example.ltd?a=a&b=b&c=c"))
self.assertEqual(queue.count_total, 1)
def setoptions(self, depth=1):
"""Define how far user want to crawl"""
options = Options()
options.scope.max_depth = depth
options.callbacks.crawler_before_start = self.crawlerstart
options.callbacks.crawler_after_finish = self.crawlerfinish
options.callbacks.request_before_start = self.requeststart
options.callbacks.request_after_finish = self.requestfinish
self.crawler = nyawcCrawler(options)
def test_regex_url_count(self):
"""Test if the amount of URLs found complies with the expected amount."""
html = ""
for url in self.__urls:
html += "\n" + url["test"]
finder = HTMLRegexLinkScraper(Options(), QueueItem(Request(""), Response()))
matches = finder.get_requests_from_content(self.__host, html)
self.assertEqual(len(matches), 30)
def test_hash_option_subdomain_must_not_match(self):
"""Ensure different subdomains are treated as one queue item if subdomains must match is False."""
options = Options()
options.scope.subdomain_must_match = False
queue = Queue(options)
queue.add_request(Request("https://www.example.ltd"))
queue.add_request(Request("https://webmail.example.ltd"))
queue.add_request(Request("https://subdomain.example.ltd"))
self.assertEqual(queue.count_total, 1)
def test_hash_option_protocol_must_not_match(self):
"""Ensure different protocols are treated as one queue item if protocols must match is False."""
options = Options()
options.scope.protocol_must_match = False
queue = Queue(options)
queue.add_request(Request("https://example.ltd"))
queue.add_request(Request("http://example.ltd"))
queue.add_request(Request("ftp://example.ltd"))
self.assertEqual(queue.count_total, 1)
def test_regex_url_matches(self):
"""Test if all the URLs match the found URLs."""
for url in self.__urls:
finder = HTMLRegexLinkScraper(Options(), QueueItem(Request(""), Response()))
requests = finder.get_requests_from_content(self.__host, url["test"])
if url["must_pass"]:
self.assertEqual(len(requests), 1)
self.assertEqual(requests[0].url, url["url"])
else:
self.assertEqual(len(requests), 0)
def test_hash_is_always_the_same(self):
"""Ensure the hashes are calculated correctly by checking for duplicates in the queue."""
options = Options()
queue = Queue(options)
for index in range(0, 100):
request = Request("https://example.ltd?1=1#2=2")
HTTPRequestHelper.patch_with_options(request, options)
request.cookies.set(name='tasty_cookie{}'.format(index), value='yum', domain='example.ltd')
queue.add_request(request)
self.assertEqual(queue.count_total, 1)
def test_crawl_website(self):
"""Crawl the website in `test/` and check if the count is correct."""
if not self.travis:
print("\n\nPlease note that the 'TestSite' unit test did not run.")
print("It will only run in Travis since it needs a webserver.")
return
options = Options()
options.callbacks.crawler_after_finish
crawler = Crawler(options)
crawler.start_with(Request("http://localhost/"))
self.assertEqual(crawler.queue.count_total, 16)
def test_soup_url_count(self):
"""Test if the amount of URLs found complies with the expected amount."""
html = ""
for url in self.__urls:
html += "\n" + url["test"]
request = Request(self.__host)
response = Response()
response.text = html
finder = HTMLSoupFormScraper(Options(), QueueItem(request, response))
matches = finder.get_requests()
self.assertEqual(len(matches), 4)
def test_soup_url_matches(self):
"""Test if all the URLs match the found URLs."""
for url in self.__urls:
request = Request(self.__host)
response = Response()
response.text = url["test"]
finder = SoupFormScraper(Options(), QueueItem(request, response))
requests = finder.get_requests()
if url["must_pass"]:
self.assertEqual(requests[0].url, url["url"])
self.assertEqual(len(requests), 1)
else:
self.assertEqual(len(requests), 0)
def main():
"""Start the scanner."""
print_banner()
setup_logger()
args = require_arguments()
options = Options()
options.scope.protocol_must_match = args.protocol_must_match
options.scope.subdomain_must_match = not args.crawl_other_subdomains
options.scope.hostname_must_match = not args.crawl_other_hostnames
options.scope.tld_must_match = not args.crawl_other_tlds
options.scope.max_depth = args.max_depth
options.performance.max_threads = args.max_threads
driver = Driver(args, options)
driver.start()
def main():
"""Start the scanner."""
print_banner()
setup_logger()
args = require_arguments()
options = Options()
options.scope.protocol_must_match = args.protocol_must_match
options.scope.subdomain_must_match = not args.scan_other_subdomains
options.scope.hostname_must_match = not args.scan_other_hostnames
options.scope.tld_must_match = not args.scan_other_tlds
options.scope.max_depth = args.max_depth if args.crawl else 0
options.performance.max_threads = args.max_threads
options.misc.verify_ssl_certificates = not args.ignore_invalid_certificates
options.misc.trusted_certificates = args.trusted_certificates
driver = Driver(args, options)
driver.start()
def __init__(self):
"""
runs when object initialized
"""
# list of crawled urls
self.crawled = []
# define steps og crawling as described above
options = Options()
# depth of crawling means depth in url tree.
# for ex apple.com and apple.com/buy has depth 1, but apple.com/buy/iphone11 has depth 2 if the only one link available
options.scope.max_depth = 1
# nothing to do before start. lambda: None is an empty action equal to def my_def(): pass
options.callbacks.crawler_before_start = lambda: None
# nothing to do after finish . lambda: None is an empty action equal to def my_def(): pass
options.callbacks.crawler_after_finish = lambda queue: None
# let nyawc to contine crawling before each request
options.callbacks.request_before_start = lambda queue, queue_item: CrawlerActions.DO_CONTINUE_CRAWLING
# call request_after_finish method after request
options.callbacks.request_after_finish = self.request_after_finish
self.crawler = nyawcCrawler(options)
def cb_request_on_error(queue_item, message):
print("[error] " + message)
def cb_form_before_autofill(queue_item, elements, form_data):
# return CrawlerActions.DO_NOT_AUTOFILL_FORM
return CrawlerActions.DO_AUTOFILL_FORM
def cb_form_after_autofill(queue_item, elements, form_data):
pass
# Declare the options
options = Options()
# Callback options (https://tijme.github.io/not-your-average-web-crawler/latest/options_callbacks.html)
options.callbacks.crawler_before_start = cb_crawler_before_start # Called before the crawler starts crawling. Default is a null route.
options.callbacks.crawler_after_finish = cb_crawler_after_finish # Called after the crawler finished crawling. Default is a null route.
options.callbacks.request_before_start = cb_request_before_start # Called before the crawler starts a new request. Default is a null route.
options.callbacks.request_after_finish = cb_request_after_finish # Called after the crawler finishes a request. Default is a null route.
options.callbacks.request_in_thread_before_start = cb_request_in_thread_before_start # Called in the crawling thread (when it started). Default is a null route.
options.callbacks.request_in_thread_after_finish = cb_request_in_thread_after_finish # Called in the crawling thread (when it finished). Default is a null route.
options.callbacks.request_on_error = cb_request_on_error # Called if a request failed. Default is a null route.
options.callbacks.form_before_autofill = cb_form_before_autofill # Called before the crawler autofills a form. Default is a null route.
options.callbacks.form_after_autofill = cb_form_after_autofill # Called after the crawler autofills a form. Default is a null route.
# Scope options (https://tijme.github.io/not-your-average-web-crawler/latest/options_crawling_scope.html)
options.scope.protocol_must_match = False # Only crawl pages with the same protocol as the startpoint (e.g. only https). Default is False.
options.scope.subdomain_must_match = True # Only crawl pages with the same subdomain as the startpoint. If the startpoint is not a subdomain, no subdomains will be crawled. Default is True.