class TestBFSNetworkCrawler(unittest.TestCase):
def setUp(self):
self.network_edges = {"11": ["1", "5", "8"], "1": ["11", "8"],
"8": ["1", "11", "5"], "5": ["11", "8"]}
self.node_info_data = None
self.node_connections_data = None
self.correct_visit_order = ["11", "5", "8", "1"]
self.crawler = None
self.STORE_TYPE = "sqlite"
def tearDown(self):
"""
Function that deletes all files after the test is complete. Disable this for testing if required.
"""
print("Deleting all data files generated as a part of test tearDown().")
self.crawler.gbuffer.destroy_stores()
self.crawler.pqueue.destroy_state()
def test_crawl(self):
# Running the bfs crawler
num_nodes_before_crash = 2
checkpoint_freq = 2
webnetwork = TestWebCaller("test", self.network_edges)
self.crawler = BFSNetworkCrawler(webnetwork, store_type=self.STORE_TYPE)
self.crawler.crawl(seed_nodes=["11", "5"], max_nodes=num_nodes_before_crash, recover=True,
checkpoint_frequency=checkpoint_freq)
node_visit_order = webnetwork.get_visit_order()
self.compare_values(node_visit_order, test_visit_order=self.correct_visit_order[:num_nodes_before_crash])
self.crawler.close()
self.crawler = BFSNetworkCrawler(webnetwork, store_type=self.STORE_TYPE)
self.crawler.crawl(seed_nodes=["11", "5"], max_nodes=10, recover=True, checkpoint_frequency=checkpoint_freq)
node_visit_order = webnetwork.get_visit_order()
print("Node visit order is", node_visit_order)
self.compare_values(node_visit_order,
test_visit_order=self.correct_visit_order)
#self.crawler.close()
def compare_values(self, real_visit_order, test_visit_order):
# Accessing the results of calling the source code
self.node_info_data = self.crawler.gbuffer.nodes_store
print(self.node_info_data)
self.node_connections_data = self.crawler.gbuffer.edges_store
self.node_visit_order = list(OrderedDict.fromkeys(real_visit_order))
# Testing the results with expected values
test_node_info = {}
test_node_edges_data = {}
#test_visit_order = ["11", "5", "8", "1"]
node_counter = 0
edge_counter = 0
test_webnetwork = TestWebCaller("test", self.network_edges)
for i in test_visit_order:
curr_node_info = test_webnetwork.get_node_info(i)
curr_node_info['id'] = node_counter
test_node_info.update({str(i): curr_node_info})
curr_edges_info = test_webnetwork.get_edges_info(i)
for edge_info in curr_edges_info:
edge_info['id'] = edge_counter
test_node_edges_data[str(edge_counter)] = edge_info
edge_counter += 1
node_counter += 1
print self.node_visit_order
pprint(self.node_info_data)
pprint(test_node_info)
pprint(self.node_connections_data)
pprint(test_node_edges_data)
#pprint(self.node_connections_data)
self.assertListEqual(test_visit_order, self.node_visit_order,
"Problem in order of BFS crawl")
self.assertDictEqual(test_node_info, dict(self.node_info_data),
"Nodes do not match: Problem in BFS crawl.")
self.assertDictEqual(test_node_edges_data, dict(self.node_connections_data),
"Edges do not match: Problem in BFS crawl.")
class TestBFSNetworkCrawler(unittest.TestCase):
def setUp(self):
self.network_edges = {"11": ["1", "5", "8"], "1": ["11", "8"], "8": ["1", "11", "5"], "5": ["11", "8"]}
self.node_info_data = None
self.node_connections_data = None
self.correct_visit_order = ["11", "5", "8", "1"]
self.crawler = None
self.STORE_TYPE = "sqlite"
def tearDown(self):
"""
Function that deletes all files after the test is complete. Disable this for testing if required.
"""
print ("Deleting all data files generated as a part of test tearDown().")
self.crawler.gbuffer.destroy_stores()
self.crawler.pqueue.destroy_state()
def test_crawl(self):
# Running the bfs crawler
num_nodes_before_crash = 2
checkpoint_freq = 2
webnetwork = TestWebCaller("test", self.network_edges)
self.crawler = BFSNetworkCrawler(webnetwork, store_type=self.STORE_TYPE)
self.crawler.crawl(
seed_nodes=["11", "5"], max_nodes=num_nodes_before_crash, recover=True, checkpoint_frequency=checkpoint_freq
)
node_visit_order = webnetwork.get_visit_order()
self.compare_values(node_visit_order, test_visit_order=self.correct_visit_order[:num_nodes_before_crash])
self.crawler.close()
self.crawler = BFSNetworkCrawler(webnetwork, store_type=self.STORE_TYPE)
self.crawler.crawl(seed_nodes=["11", "5"], max_nodes=10, recover=True, checkpoint_frequency=checkpoint_freq)
node_visit_order = webnetwork.get_visit_order()
print ("Node visit order is", node_visit_order)
self.compare_values(node_visit_order, test_visit_order=self.correct_visit_order)
# self.crawler.close()
def compare_values(self, real_visit_order, test_visit_order):
# Accessing the results of calling the source code
self.node_info_data = self.crawler.gbuffer.nodes_store
print (self.node_info_data)
self.node_connections_data = self.crawler.gbuffer.edges_store
self.node_visit_order = list(OrderedDict.fromkeys(real_visit_order))
# Testing the results with expected values
test_node_info = {}
test_node_edges_data = {}
# test_visit_order = ["11", "5", "8", "1"]
node_counter = 0
edge_counter = 0
test_webnetwork = TestWebCaller("test", self.network_edges)
for i in test_visit_order:
curr_node_info = test_webnetwork.get_node_info(i)
curr_node_info["id"] = node_counter
test_node_info.update({str(i): curr_node_info})
curr_edges_info = test_webnetwork.get_edges_info(i)
for edge_info in curr_edges_info:
edge_info["id"] = edge_counter
test_node_edges_data[str(edge_counter)] = edge_info
edge_counter += 1
node_counter += 1
print self.node_visit_order
pprint(self.node_info_data)
pprint(test_node_info)
pprint(self.node_connections_data)
pprint(test_node_edges_data)
# pprint(self.node_connections_data)
self.assertListEqual(test_visit_order, self.node_visit_order, "Problem in order of BFS crawl")
self.assertDictEqual(test_node_info, dict(self.node_info_data), "Nodes do not match: Problem in BFS crawl.")
self.assertDictEqual(
test_node_edges_data, dict(self.node_connections_data), "Edges do not match: Problem in BFS crawl."
)
