def test_crawl(self):
crawler = MassCrawl(seeds=self.unicode_seeds)
crawler.crawl(num_threads=4, time_per_url=5, request_timeout=3,
proxy_list=[{}])
accumulated_target_urls = [x.url for x in crawler.targets]
# The right way
#self.assertEqual(baseline_target_urls, accumulated_target_urls)
# The very not right way to work around inconsistent results from live targets
# assert lists have an overlap that is within MARGIN items of the baseline.
intersections = set(baseline_target_urls).intersection(accumulated_target_urls)
intersect_length = len(intersections)
base_length = len(baseline_target_urls)
self.assertAlmostEqual(intersect_length, base_length, delta=MARGIN)
def mapper(output_file):
seeds = get_stdin()
mc = MassCrawl(seeds)
logger.info("Adding seeds:")
for seed in seeds:
logger.info(mc.get_domain_from_url(seed))
mc.add_to_scope_from_url(seed)
mc.crawl(depth=3, num_threads=25, time_per_url=5, request_timeout=4, proxy_list=proxy_scan_list)
if mc.targets:
results = fuzz(mc.targets)
for result in results:
domain = mc.get_domain_from_url(result.fuzzy_target.url)
output_file.write("domain: %s \\\\ results: %s\n" % (domain, result))
else:
logger.error("URL had no mc.targets for some reason... dump of seeds: %s", seeds)
output_file.write("domain: %s \\\\ ERROR: no targets\n" % domain)
