class TestDataCollection(unittest.TestCase):
def setUp(self):
self.start_url = "https://www.goodreads.com/book/show/53175355-many-points-of-me"
self.connection_string = os.getenv('MONGO_CONNECTION_STRING')
self.testDB = DataCollection(self.connection_string, "testDatabase",
"testCollection")
def testPushToBookCollection(self):
self.testDB.empty_data_collection()
test = {"url": 1, "test": 2}
self.testDB.push_to_collection(test)
self.assertEqual(True, self.testDB.document_already_exist(test))
def testempty_data_collection(self):
self.testDB.empty_data_collection()
self.assertEqual(0, self.testDB.get_collection_size())
def testget_collection_size(self):
self.testDB.empty_data_collection()
test1 = {"url": 3, "test": 2}
test2 = {"url": 1, "test": 1}
self.testDB.push_to_collection(test1)
self.testDB.push_to_collection(test2)
self.assertEqual(2, self.testDB.get_collection_size())
def testdocument_already_exist(self):
self.testDB.empty_data_collection()
test1 = {"url": 3, "test": 2}
test2 = {"url": 1, "test": 1}
self.testDB.push_to_collection(test1)
self.assertEqual(True, self.testDB.document_already_exist(test1))
self.assertEqual(False, self.testDB.document_already_exist(test2))
def clear_database(data_collection_type):
"""Clear the specified data collection in the database
Args:
data_collection_type (str): Name of data collection, either 'book' or 'author'
"""
collection_name = data_collection_type
if collection_name not in ('book', 'author'):
print("Error: no collection named " + data_collection_type +
", please enter 'book' or 'author' ")
return
database = DataCollection(MONGO_CONNECTION_STRING, "goodReads",
collection_name)
database.empty_data_collection()
class TestScraper(unittest.TestCase):
def setUp(self):
self.testDB = DataCollection(os.getenv('MONGO_CONNECTION_STRING'),
"testDatabase", "testCollection")
self.bookScraper = BookScraper(self.testDB)
self.authroScraper = AuthorScraper(self.testDB)
def testBookScraper(self):
self.testDB.empty_data_collection()
testurl = "https://www.goodreads.com/book/show/6185.Wuthering_Heights"
self.bookScraper.scrape_one_book(testurl)
self.assertEqual(1, self.testDB.get_collection_size())
def testAuthorScraper(self):
self.testDB.empty_data_collection()
testurl = "https://www.goodreads.com/author/show/6485178.Fredrik_Backman"
self.authroScraper.scrape_one_author(testurl)
self.assertEqual(1, self.testDB.get_collection_size())