def make_map(arg_tuple):
dynamic_path = "/usr/lib/chromium-browser/chromedriver"
url = arg_tuple[0]
max_nodes = arg_tuple[1]
dynamic = arg_tuple[2]
url_map = UrlMap(url, dynamic_path, url, dynamic_pages=dynamic)
url_map.create_map(total_iterations=max_nodes)
return url_map
from urlmapper import UrlMap import time import json from operator import itemgetter mypath = "/Users/Tilley/Downloads/chromedriver" myurl = "https://sjrfire.com" myurlstart = "https://sjrfire.com" # myurl = "https://youtube.com" # myurl = "https://reddit.com" # creates our UrlMap object url_map = UrlMap(myurl, mypath, myurlstart, dynamic_pages=False) url_map.create_map(total_iterations=5) print(url_map.this_map)
import unittest
import sys
sys.path.append("..")
from urlmapper import UrlMap
myurl1 = "https://google.com/"
myurl2 = "https://reddit.com/"
mypath = "/Users/Tilley/Downloads/chromedriver"
mymap1 = UrlMap(myurl1, mypath, myurl1, dynamic_pages=False)
mymap1.create_map(total_iterations=10)
mymap2 = UrlMap(myurl2, mypath, myurl2)
mymap2.create_map(total_iterations=0)
mymap3 = UrlMap(myurl1, mypath)
mymap3.create_map() # total_iterations defaults to 30
# print(mymap1.get_map())
# print(len(mymap1.this_map.keys()))
class TestUrlMap(unittest.TestCase):
# test to see that we get the correct amount of nodes
def test_map_max_nodes(self):
self.assertEqual(len(mymap1.this_map.keys()), 10)
self.assertEqual(len(mymap2.get_map().keys()), 0)
self.assertEqual(len(mymap3.this_map.keys()), 30)
# -1 goes forever and cannot be tested
# test to see that our queue remains intact after termination
