def test_insert(self): # Given fixture = ShopFixture() fixture.create2Shops() givenShop01 = fixture.shops[0] givenShop02 = fixture.shops[1] sut = TinyShopDao(self.testDBPath) sut.connection.open() # When sut.insert(data=givenShop01) # Then allShops = sut.loadAll() self.assertIsInstance(allShops, list) self.assertEqual(1, len(allShops)) self.assertEqual(givenShop01, allShops[0]) # When sut.insert(data=givenShop02) # Then # We use client code here for convenience to convert saved data back to # Shop objects, which is not perfect. allShops = sut.loadAll() self.assertIsInstance(allShops, list) self.assertEqual(2, len(allShops)) self.assertEqual(givenShop01, allShops[0]) self.assertEqual(givenShop02, allShops[1]) sut.connection.close()
def test_find_shouldRaiseOnWrongKeywords(self): # Given # When / Then with TinyShopDao(self.testDBPath) as sut: with self.assertRaises(KeyError): sut.find(invalidKeyword="Value does not matter here")
class ConcreteShopHelper: """ Helper for shop scraping tests. Is able to create a fully scrapable concrete shop. Able to create this shop as a record within a temporary shop repository. """ shop: Shop shopDaoPath = TEMP_SHOPS_TINYDB_TEST_PATH shopDao = TinyShopDao(path=shopDaoPath) shopRepo = ShopRepo(dao=shopDao) productsUrlsRepo: ProductsUrlsRepo def __init__(self, shop: Shop, productsUrlsRepoPath: Path): self.shop = shop self.productsUrlsRepoPath = productsUrlsRepoPath assert productsUrlsRepoPath.is_file() productsUrlsDao = ProductsUrlsDao(productsUrlsRepoPath) self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao) def assignProductsFromProductsUrlsRepo(self): # Generate shop products from products URLs (which are fixtures). products = self.productsUrlsRepo.getAll() assert len(products) > 0 if products: self.shop.assignProducts(products) assert len(self.shop.products) > 0 def overwriteRepoTableWithScrapableShop(self): # Create fresh 'Shops' table with this shop as its data. self.assignProductsFromProductsUrlsRepo() self.shopRepo.setAll(shops=[self.shop])
def test_findByName(self): # Given # Create test data to search for. We use two shops with the same name here. shopData1 = dict(uid="ca0f5926-7d55-4973-a8e1-d3e2cc89fca6", name="Shop with same name") shopData2 = dict(uid="e68782fd-19af-428e-881f-99d7af9b83b0", name="This shop should not be found") shopData3 = dict(uid="b0e2e467-6fd5-4a06-bb1e-9ad60223cafa", name="Shop with same name") expectedShops = [Shop(**shopData1), Shop(**shopData3)] with tdb.TinyDB(self.testDBPath) as db: shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME) shopTable.insert(shopData1) shopTable.insert(shopData2) shopTable.insert(shopData3) # Setup repo testTinyShopDao = TinyShopDao(path=self.testDBPath) sut = ShopRepo(dao=testTinyShopDao) # When foundShops = sut.findByName("Shop with same name") # Then self.assertIsInstance(foundShops, list) self.assertEqual(2, len(foundShops)) self.assertEqual(expectedShops, foundShops)
def test_findByUID(self): # Given # Create test data to search for. uidToFind = "b0e2e467-6fd5-4a06-bb1e-9ad60223cafa" shopData1 = dict(uid="ca0f5926-7d55-4973-a8e1-d3e2cc89fca6", name="The name of the first test shop") shopData2 = dict(uid=uidToFind, name="The name of the second test shop") expectedShop = Shop(**shopData2) with tdb.TinyDB(self.testDBPath) as db: shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME) shopTable.insert(shopData1) shopTable.insert(shopData2) # Setup repo testTinyShopDao = TinyShopDao(path=self.testDBPath) sut = ShopRepo(dao=testTinyShopDao) # When foundShop = sut.findByUID(uidToFind) # Then self.assertIsInstance(foundShop, Shop) self.assertEqual(foundShop.uid, uidToFind) self.assertEqual(expectedShop, foundShop)
def __init__(self): self.logfilePath = APP_USERDATA_DIR / "Logs/log_current.txt" self.shopsRepoPath = APP_USERDATA_DIR / "Shops.json" shopDao = TinyShopDao(path=self.shopsRepoPath) self.productsUrlsRepoPath = APP_USERDATA_DIR / "ProductsURLs.txt" productsUrlsDao = ProductsUrlsDao(filepath=self.productsUrlsRepoPath) self.messengersRepoPath = APP_USERDATA_DIR / "Messengers.json" discordMessengerDao = msn.DiscordTinyDao(path=self.messengersRepoPath) self.proxiesRepoPath = APP_USERDATA_DIR / "Proxies.txt" proxyDao = FileProxyDao(filepath=self.proxiesRepoPath) self.userAgentsRepoPath = APP_USERDATA_DIR / "UserAgents.txt" userAgentDao = FileUserAgentDao(filepath=self.userAgentsRepoPath) self.shopRepo = ShopRepo(dao=shopDao) self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao) self.discordMessengerRepo = msn.Repo(dao=discordMessengerDao) self.proxyRepo = ProxyRepo(dao=proxyDao) self.userAgentRepo = UserAgentRepo(dao=userAgentDao) self.session = None self.scrapers: List[Scraper] = list() self.shops: List[Shop] = list() self._configureLogger() self._createRepoFilesIfNotExist()
def test_update_shouldRaiseIfGivenDataHasWrongType(self): # Given invalidData = Mock(name="invalidType") # When / Then with TinyShopDao(self.testDBPath) as sut: with self.assertRaises(TypeError): sut.update(data=invalidData)
def test_find_uid(self): # Given # Generate test-data from 2 shop fixtures fixture = ShopFixture() fixture.create2Shops() expectedShop = fixture.shops[1] expectedUID = fixture.shops[1].uid # UID of second fixture shop with TinyShopDao(self.testDBPath) as dao: dao.saveAll(data=fixture.shops) # When with TinyShopDao(self.testDBPath) as sut: foundShop = sut.find(uid=expectedUID) # Then self.assertIsInstance(foundShop, Shop) self.assertEqual(expectedShop, foundShop)
def test_find_name(self): # Given # Generate test-data from 2 shop fixtures fixture = ShopFixture() fixture.create2Shops() expectedShop = fixture.shops[0] expectedShopName = fixture.shops[ 0].name # shop name of second fixture shop with TinyShopDao(self.testDBPath) as dao: dao.saveAll(data=fixture.shops) # When with TinyShopDao(self.testDBPath) as sut: foundShops = sut.find(shopName=expectedShopName) # Then self.assertIsInstance(foundShops, list) self.assertEqual(1, len(foundShops)) self.assertIsInstance(foundShops[0], Shop) self.assertEqual(foundShops, [expectedShop])
class TempShopRepoHelper: """ Helper for creating a controlling a fully scrapable temporary TinyDB shop repo. The repository is created at initialization. It uses a path to a fixture file. The shops table is overridden with data passed by the 'shops' argument at initialization. """ shopDaoPath: ClassVar[Path] = TEMP_SHOPS_TINYDB_TEST_PATH shopDao: ClassVar[TinyShopDao] = TinyShopDao(path=shopDaoPath) shopRepo: ClassVar[ShopRepo] = ShopRepo(dao=shopDao) def __init__(self, shops: List[Shop]): self.shops = shops self.shopRepo.setAll(shops=self.shops)
def test_loadAll(self): # Given assert self.testDBPath.exists() # Precondition for the test fixture = ShopFixture() fixture.create2Shops() # Generate test-data sut = TinyShopDao(self.testDBPath) sut.connection.open() sut.saveAll(data=fixture.shops) # When # We use client code here for convenience to convert saved data back to # Shop objects, which is not perfect. loadedShops = sut.loadAll() sut.connection.close() # Then self.assertIsInstance(loadedShops, list) self.assertEqual(2, len(loadedShops)) self.assertEqual(fixture.shops[0], loadedShops[0]) self.assertEqual(fixture.shops[1], loadedShops[1])
def test_update(self): # Given assert self.testDBPath.exists() # Precondition for the test # Generate test-data with 2 shop fixtures fixture = ShopFixture() fixture.create2Shops() fixture.shops[1].uid = "69ec8e1b-8812-4413-ad72-b74364e2fa7a" with TinyShopDao(self.testDBPath) as sut: sut.saveAll(data=fixture.shops) # Data to update. Note we set reference values here, so the original fixture # objects get modified, which is important for later equality checks. expectedShop = fixture.shops[1] expectedShop.name = "An updated shop name" expectedShop.url = "https://updated-shop-url.com" expectedShop.products[0].name = "An updated product name" expectedShop.products[0].sizes[0].url = "https://updated-size.com" # When with TinyShopDao(self.testDBPath) as sut: sut.update(data=expectedShop) # Then # Expect that DB still exists self.assertTrue(self.testDBPath.exists()) # We use client code here for convenience to convert saved data back to # Shop objects, which is not perfect. with TinyShopDao(self.testDBPath) as sut: savedShops = sut.loadAll() # Expect that the first shop has been left untouched. self.assertEqual(fixture.shops[0], savedShops[0]) # Expect that the second shop has been updated with correct updated values. self.assertEqual(fixture.shops[1], savedShops[1])
def test_getAll(self): # Given testTinyShopDao = TinyShopDao(path=self.testDBPath) # Create 2 shops in TinyDB for testing. # Note that we use client code to create them, which is more of an integration test... fixture = ShopFixture() fixture.create2Shops() expectedShops = fixture.shops ShopRepo(dao=testTinyShopDao).setAll(shops=expectedShops) sut = ShopRepo(dao=testTinyShopDao) # When loadedShops = sut.getAll() # Then # Expect that loaded shops match the expected self.assertEqual(expectedShops, loadedShops)
def test_update(self): # Given # Create 2 shops in TinyDB for testing. fixture = ShopFixture() fixture.create2Shops() expectedShop = fixture.shops[0] assert expectedShop.uid is not None and expectedShop.uid != "" # Write a shop which we can try to update by UID. existingData = dict(uid=expectedShop.uid, name="I don't know this shop's name") with tdb.TinyDB(self.testDBPath) as db: shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME) shopTable.insert(existingData) # Setup repo testTinyShopDao = TinyShopDao(path=self.testDBPath) sut = ShopRepo(dao=testTinyShopDao) # When sut.update(shop=expectedShop) # Then with tdb.TinyDB(self.testDBPath) as db: shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME) recordList: list = shopTable.all() self.assertEqual(1, len(recordList)) # Expect that data with previous uid still exist self.assertEqual(expectedShop.uid, recordList[0].get("uid")) # Expect that shop's name has been updated self.assertNotEqual("I don't know this shop's name", recordList[0].get("name")) # Note that we use client code to load the shop again, which is # more of an integration test... updatedShops = sut.getAll() self.assertIsInstance(updatedShops, list) self.assertEqual(1, len(recordList)) # Expect that updated shop matches the expected one self.assertEqual(expectedShop, updatedShops[0])
def test_setAll(self): # Given # Insert a document into a fresh 'Shops' table. This data is expected to be # completely overridden by the test. existingData = dict(OneTestOne="Test data val 1", TwoTestTwo="Test data val 2") with tdb.TinyDB(self.testDBPath) as db: shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME) shopTable.insert(existingData) # These data are expected: fixture = ShopFixture() fixture.create2Shops() expectedShops = fixture.shops # Setup repo testTinyShopDao = TinyShopDao(path=self.testDBPath) sut = ShopRepo(dao=testTinyShopDao) # When sut.setAll(shops=expectedShops) # Then with tdb.TinyDB(self.testDBPath) as db: shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME) recordList: list = shopTable.all() # Expect that previous data do not exist anymore self.assertLessEqual(0, len(recordList)) self.assertIsNone(recordList[0].get("OneTestOne")) self.assertIsNone(recordList[0].get("TwoTestTwo")) # Note that we use client code to load the shops again, which is # more of an integration test... loadedShops = sut.getAll() # Expect that loaded shops match the expected ones self.assertEqual(expectedShops, loadedShops)
def test_updateFromProductsUrls(self): # Given # Copy fixture to new arbitrary file as we will modify its contents within this test. with open(str(PRODUCTS_URLS_9_VALID_TEST_PATH), "r", encoding="utf-8") as source: content = source.read() with open(str(self.tempProductsUrlsRepoPath), "w+", encoding="utf-8") as target: target.write(content) # Note that the table gets deleted by the unit test's setup() method - so we # start with a fresh empty table. testTinyShopDao = TinyShopDao(path=self.testDBPath) sut = ShopRepo(dao=testTinyShopDao) productsUrlsRepo = ProductsUrlsRepoMock( productsUrlsRepoPath=self.tempProductsUrlsRepoPath) expectedProducts = productsUrlsRepo.getAll() expectedProductUrls = [p.url for p in expectedProducts] # 1. Test initial update ----------------------------------------------------------- # When # This is expected to fill the table with all the fixture data of ProductsUrls repo. sut.updateFromProductsUrls(productsUrlsRepo=productsUrlsRepo) # Then shops = sut.getAll() self.assertIsInstance(shops, list) self.assertEqual(3, len(shops)) # Expect that all shops have been inserted shopsUrls = [s.url for s in shops] self.assertIn("https://www.solebox.com", shopsUrls) self.assertIn("http://real.fantastic.de", shopsUrls) self.assertIn("https://www.dbyte.org", shopsUrls) # Expect that all products have been inserted soleboxShop = list( filter(lambda s: s.url == "https://www.solebox.com", shops))[0] self.assertIsInstance(soleboxShop.products, list) self.assertEqual(1, len(soleboxShop.products)) for product in soleboxShop.products: self.assertIn(product.url, expectedProductUrls) realFantasticShop = list( filter(lambda s: s.url == "http://real.fantastic.de", shops))[0] self.assertIsInstance(realFantasticShop.products, list) self.assertEqual(2, len(realFantasticShop.products)) for product in realFantasticShop.products: self.assertIn(product.url, expectedProductUrls) dbyteShop = list( filter(lambda s: s.url == "https://www.dbyte.org", shops))[0] self.assertIsInstance(dbyteShop.products, list) self.assertEqual(6, len(dbyteShop.products)) for product in dbyteShop.products: self.assertIn(product.url, expectedProductUrls) # 2. Test delete product/shop ----------------------------------------------------- # Given # Remove all http://real.fantastic.de/... URLs from ProductsUrls repo. with open(str(self.tempProductsUrlsRepoPath), "r+", encoding="utf-8") as target: lines = target.readlines() for line in reversed(lines): if line.startswith( "http://real.fantastic.de/shop/great-realdumbtrump.htm" ): lines.remove(line) if line.startswith( "http://real.fantastic.de/shop/buy-new-holo?prodid=682357ac" ): lines.remove(line) # Overwrite file with the updated data target.seek(0) target.writelines(lines) # When # This is expected to remove shop http://real.fantastic.de entirely from database, # because it's products do not exist anymore in ProductsUrls repo. sut.updateFromProductsUrls(productsUrlsRepo=productsUrlsRepo) # Then shops = sut.getAll() self.assertIsInstance(shops, list) self.assertEqual(2, len(shops)) # Expect that shop http://real.fantastic.de has been entirely removed from database realFantasticShop = list( filter(lambda s: s.url == "http://real.fantastic.de", shops)) self.assertIsInstance(realFantasticShop, list) self.assertEqual(0, len(realFantasticShop)) # 3. Test add product to existing shop ---------------------------------------------- # Given with open(str(self.tempProductsUrlsRepoPath), "r+", encoding="utf-8") as target: lines = target.readlines() lines.append("\nhttps://www.solebox.com/some-new-product\n") # Overwrite file with the updated data target.seek(0) target.writelines(lines) expectedProducts = productsUrlsRepo.getAll() expectedProductUrls = [p.url for p in expectedProducts] # When # This is expected to update shop https://www.solebox.com with the above added # product https://www.solebox.com/some-new-product sut.updateFromProductsUrls(productsUrlsRepo=productsUrlsRepo) # Then shops = sut.getAll() self.assertIsInstance(shops, list) self.assertEqual(2, len(shops)) # Expect that product https://www.solebox.com/some-new-product has been added to # existing shop with URL https://www.solebox.com soleboxShop = list( filter(lambda s: s.url == "https://www.solebox.com", shops))[0] self.assertIsInstance(soleboxShop.products, list) self.assertEqual(2, len(soleboxShop.products)) for product in soleboxShop.products: self.assertIn(product.url, expectedProductUrls) # 4. Test add shop to existing shops ------------------------------------------------- # Given with open(str(self.tempProductsUrlsRepoPath), "r+", encoding="utf-8") as target: lines = target.readlines() lines.append("\nhttps://new-shop-1833663.com/new-product.htm\n") # Overwrite file with the updated data target.seek(0) target.writelines(lines) expectedProducts = productsUrlsRepo.getAll() expectedProductUrls = [p.url for p in expectedProducts] # When # This is expected to update the shop table (which already has shops in it) with # the above added product which has a base url which currently not exists # in the shops table. So a new shop with this product must be created in shopRepo. sut.updateFromProductsUrls(productsUrlsRepo=productsUrlsRepo) # Then shops = sut.getAll() self.assertIsInstance(shops, list) self.assertEqual(3, len(shops)) # Expect that shop https://new-shop-1833663.com has been added to # existing database. newShop = list( filter(lambda s: s.url == "https://new-shop-1833663.com", shops))[0] self.assertIsInstance(newShop.products, list) self.assertEqual(1, len(newShop.products)) for product in newShop.products: self.assertIn(product.url, expectedProductUrls)
def test_saveAll(self): # Given assert self.testDBPath.exists() # Precondition for the test fixture = ShopFixture() fixture.create2Shops() # When with TinyShopDao(self.testDBPath) as sut: sut.saveAll(data=fixture.shops) # Then # Expect that DB still exists self.assertTrue(self.testDBPath.exists()) # Get a ref to the DB for further tests dbRef = tdb.TinyDB(str(self.testDBPath)) shopsTable: tdb.database.Table = dbRef.table(TinyShopDao._TABLE_NAME) # Expect that 'Shops' table has 2 records self.assertEqual(2, shopsTable.count(cond=lambda x: True)) # 1st SHOP # ------------------------------------------------------------------------ # Expect that the first shop's attribute values are correct self.assertEqual("73f9cac8-ebdc-4d9b-8163-d04d09f06cd9", shopsTable.get(doc_id=1).get("uid")) self.assertEqual("Bottle shop", shopsTable.get(doc_id=1).get("name")) self.assertEqual("http://oneshop.com/bottles", shopsTable.get(doc_id=1).get("url")) # Expect that the first shop has a list of products products = shopsTable.get(doc_id=1).get("products") self.assertIsInstance(products, list) # Expect that the first shop has 2 products self.assertEqual(2, len(products)) # 1st SHOP, 1st Product # ------------------------------------------------------------------------ # Expect correct values of the first product product = products[0] self.assertEqual("2857027b-cf25-4639-965e-0e22f9f4c755", product.get("uid")) self.assertEqual("Biggest Corona Bottle ever", product.get("name")) self.assertEqual("http://oneshop.com/bottles/92743867ACTFGJ-UTU", product.get("url")) self.assertEqual(None, product.get("urlThumb")) self.assertEqual(55.49, product.get("basePrice")) self.assertEqual(1588548868.304869, product.get("lastScanStamp")) self.assertEqual(1601466659.0, product.get("releaseDateStamp")) # Expect correct sizes of the first product sizes = product.get("sizes") self.assertEqual(2, len(sizes)) size = sizes[0] self.assertEqual("1528dae6-188f-4d7e-8a6c-5af44ce5c222", size.get("uid")) self.assertEqual("40 1/3", size.get("sizeEU")) self.assertEqual(56.99, size.get("price")) self.assertEqual( "http://oneshop.com/bottles/92743867ACTFGJ-UTU/40.1.3.htm", size.get("url")) self.assertEqual( "http://oneshop.com/bottles/atc/40.1.3-92743867ACTFGJ-UTU.htm", size.get("urlAddToCart")) self.assertEqual(True, size.get("isInStock")) size = sizes[1] self.assertEqual("5f561c62-8502-4ec1-8f46-f0adb5e8254c", size.get("uid")) self.assertEqual("43", size.get("sizeEU")) self.assertEqual(54.99, size.get("price")) self.assertEqual( "http://oneshop.com/bottles/92743867ACTFGJ-UTU/43.htm", size.get("url")) self.assertEqual( "http://oneshop.com/bottles/atc/43-92743867ACTFGJ-UTU.htm", size.get("urlAddToCart")) self.assertEqual(False, size.get("isInStock")) # 1st SHOP, 2nd Product # ------------------------------------------------------------------------ # Expect correct values of the second product product = products[1] self.assertEqual("9cab557a-419a-4883-8287-f09f7244b225", product.get("uid")) self.assertEqual("Neck Bottle", product.get("name")) self.assertEqual("http://oneshop.com/bottles/1362836400447GT-UTU", product.get("url")) self.assertEqual(None, product.get("urlThumb")) self.assertEqual(3.22, product.get("basePrice")) self.assertEqual(1588548911.230381, product.get("lastScanStamp")) self.assertEqual(1675089635.0, product.get("releaseDateStamp")) # Expect that the second product has an empty sizes list sizes = product.get("sizes") self.assertIsInstance(sizes, list) self.assertEqual(0, len(sizes)) # 2nd SHOP # ------------------------------------------------------------------------ # Expect that the second shop's attribute values are correct self.assertEqual("69ec8e1b-8812-4413-ad72-b74364e2fa7a", shopsTable.get(doc_id=2).get("uid")) self.assertEqual("Megashop", shopsTable.get(doc_id=2).get("name")) self.assertEqual("https://www.megashop.com/shoes", shopsTable.get(doc_id=2).get("url")) # 2nd SHOP, 1st Product # ------------------------------------------------------------------------ # Expect that the second shop has a list of products products = shopsTable.get(doc_id=2).get("products") self.assertIsInstance(products, list) # Expect that the second shop has 1 product self.assertEqual(1, len(products)) # Expect correct values of the first product product = products[0] self.assertEqual("f0700293-693c-48a6-8f01-014e07151d99", product.get("uid")) self.assertEqual("Hey Bro Male", product.get("name")) self.assertEqual("https://www.megashop.com/shoes/9a734hd78.html", product.get("url")) self.assertEqual("https://www.megashop.com/shoes/thumb-9a734hd78.html", product.get("urlThumb")) self.assertEqual(190, product.get("basePrice")) self.assertEqual(1588548274.102859, product.get("lastScanStamp")) self.assertEqual(1856674960.0, product.get("releaseDateStamp")) # Expect correct sizes of the first product sizes = product.get("sizes") self.assertEqual(1, len(sizes)) size = sizes[0] self.assertEqual("e070b0c9-769d-4c13-a208-f7207f0970db", size.get("uid")) self.assertEqual("44.5", size.get("sizeEU")) self.assertEqual(189.5, size.get("price")) self.assertEqual("https://megashop.com/shoes/44.5-9a734hd78.htm", size.get("url")) self.assertEqual( "https://megashop.com/shoes/atc/44.5-9a734hd78#g89.php", size.get("urlAddToCart")) self.assertEqual(True, size.get("isInStock")) # Cleanup dbRef.close()