예제 #1
0
파일: urlsaver.py 프로젝트: xurenlu/hyer
 def test_save(self):
     """ test if the url saver works as expected"""
     url_db = url_saver.url_saver()
     url_db.add("http://www.sohu.com/", "task1")
     url_db.add("http://www.sina.com/2008/", "task1")
     url_db.add("http://2008.sina.com/", "task1")
     url_db.save_to("/tmp/_hyer.test.1954")
     b_urls_queue = url_db.urls_queue
     b_visited_urls = url_db.visited_urls
     url_db.load_from("/tmp/_hyer.test.1954")
     self.assertEqual(b_urls_queue, url_db.urls_queue)
     self.assertEqual(b_visited_urls, url_db.visited_urls)
예제 #2
0
 def test_save(self):
     ''' test if the url saver works as expected'''
     url_db = url_saver.url_saver()
     url_db.add("http://www.sohu.com/", "task1")
     url_db.add("http://www.sina.com/2008/", "task1")
     url_db.add("http://2008.sina.com/", "task1")
     url_db.save_to("/tmp/_hyer.test.1954")
     b_urls_queue = url_db.urls_queue
     b_visited_urls = url_db.visited_urls
     url_db.load_from("/tmp/_hyer.test.1954")
     self.assertEqual(b_urls_queue, url_db.urls_queue)
     self.assertEqual(b_visited_urls, url_db.visited_urls)
예제 #3
0
파일: spider.py 프로젝트: lennon310/hyer
	def test_pop(self):
		us=url_saver.url_saver()
		us.add("http://www.sina.com/","task1")
		us.add("http://www.sohu.com/","task1")
		url=us.pop("task1")
		#self.assertEqual(url,"http://www.sohu.com/")	
		url=us.pop("task2")
		#self.assertEqual(url,"http://www.sina.com/")	
		us.mark("http://www.sina.com/","task1")
		if_visited=us.if_visited("http://www.sina.com/","task1")
		self.assertEqual(if_visited,1)
		if_visited=us.if_visited("http://www.sohu.com/","task1")
		self.assertEqual(if_visited,0)
예제 #4
0
 def test_pop(self):
     us = url_saver.url_saver()
     us.add("http://www.sina.com/", "task1")
     us.add("http://www.sohu.com/", "task1")
     url = us.pop("task1")
     #self.assertEqual(url,"http://www.sohu.com/")
     url = us.pop("task2")
     #self.assertEqual(url,"http://www.sina.com/")
     us.mark("http://www.sina.com/", "task1")
     if_visited = us.if_visited("http://www.sina.com/", "task1")
     self.assertEqual(if_visited, 1)
     if_visited = us.if_visited("http://www.sohu.com/", "task1")
     self.assertEqual(if_visited, 0)
예제 #5
0
파일: urlsaver.py 프로젝트: xurenlu/hyer
 def test_pop(self):
     url_db = url_saver.url_saver()
     url_db.add("http://www.sohu.com/", "task1")
     url_db.add("http://www.sina.com/2008/", "task1")
예제 #6
0
 def test_pop(self):
     url_db = url_saver.url_saver()
     url_db.add("http://www.sohu.com/", "task1")
     url_db.add("http://www.sina.com/2008/", "task1")