def test_createShops_shouldRaiseIfNoValidURLsFound(self):
# Given
emptyUrlsDao = ProductsUrlsDao(
filepath=PRODUCTS_URLS_0_VALID_TEST_PATH)
sut = ProductsUrlsRepo(dao=emptyUrlsDao)
# When
with self.assertRaises(LookupError):
sut.createShops()
def test_getAll(self):
# Given
testUrlsDao = ProductsUrlsDao(filepath=PRODUCTS_URLS_9_VALID_TEST_PATH)
sut = ProductsUrlsRepo(dao=testUrlsDao)
# When
products = sut.getAll()
# Then
self.assertIsInstance(products, list)
self.assertTrue(
all(isinstance(i, Product) for i in products),
"Expected that all list elements are of type 'Product'")
self.assertEqual(9, len(products))
def test_createShops(self):
# Given
testUrlsDao = ProductsUrlsDao(filepath=PRODUCTS_URLS_9_VALID_TEST_PATH)
sut = ProductsUrlsRepo(dao=testUrlsDao)
# When
shops: List[Shop] = sut.createShops()
# Then
self.assertIsInstance(shops, list)
self.assertEqual(3, len(shops))
shopsUrls = [s.url for s in shops]
self.assertIn("https://www.solebox.com", shopsUrls)
self.assertIn("https://www.dbyte.org", shopsUrls)
self.assertIn("http://real.fantastic.de", shopsUrls)
def __init__(self):
self.logfilePath = APP_USERDATA_DIR / "Logs/log_current.txt"
self.shopsRepoPath = APP_USERDATA_DIR / "Shops.json"
shopDao = TinyShopDao(path=self.shopsRepoPath)
self.productsUrlsRepoPath = APP_USERDATA_DIR / "ProductsURLs.txt"
productsUrlsDao = ProductsUrlsDao(filepath=self.productsUrlsRepoPath)
self.messengersRepoPath = APP_USERDATA_DIR / "Messengers.json"
discordMessengerDao = msn.DiscordTinyDao(path=self.messengersRepoPath)
self.proxiesRepoPath = APP_USERDATA_DIR / "Proxies.txt"
proxyDao = FileProxyDao(filepath=self.proxiesRepoPath)
self.userAgentsRepoPath = APP_USERDATA_DIR / "UserAgents.txt"
userAgentDao = FileUserAgentDao(filepath=self.userAgentsRepoPath)
self.shopRepo = ShopRepo(dao=shopDao)
self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao)
self.discordMessengerRepo = msn.Repo(dao=discordMessengerDao)
self.proxyRepo = ProxyRepo(dao=proxyDao)
self.userAgentRepo = UserAgentRepo(dao=userAgentDao)
self.session = None
self.scrapers: List[Scraper] = list()
self.shops: List[Shop] = list()
self._configureLogger()
self._createRepoFilesIfNotExist()
class ConcreteShopHelper:
""" Helper for shop scraping tests. Is able to create a fully scrapable
concrete shop. Able to create this shop as a record within a temporary shop repository. """
shop: Shop
shopDaoPath = TEMP_SHOPS_TINYDB_TEST_PATH
shopDao = TinyShopDao(path=shopDaoPath)
shopRepo = ShopRepo(dao=shopDao)
productsUrlsRepo: ProductsUrlsRepo
def __init__(self, shop: Shop, productsUrlsRepoPath: Path):
self.shop = shop
self.productsUrlsRepoPath = productsUrlsRepoPath
assert productsUrlsRepoPath.is_file()
productsUrlsDao = ProductsUrlsDao(productsUrlsRepoPath)
self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao)
def assignProductsFromProductsUrlsRepo(self):
# Generate shop products from products URLs (which are fixtures).
products = self.productsUrlsRepo.getAll()
assert len(products) > 0
if products:
self.shop.assignProducts(products)
assert len(self.shop.products) > 0
def overwriteRepoTableWithScrapableShop(self):
# Create fresh 'Shops' table with this shop as its data.
self.assignProductsFromProductsUrlsRepo()
self.shopRepo.setAll(shops=[self.shop])
def __init__(self):
self.productsUrlsRepoPath = PRODUCTS_URLS_INTEGRATION_TEST_PATH
assert self.productsUrlsRepoPath.is_file()
productsUrlsDao = ProductsUrlsDao(filepath=self.productsUrlsRepoPath)
self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao)
super().__init__(dao=productsUrlsDao)
def test_init_shouldSetDefaultValues(self):
# When
daoMock = Mock()
daoMock.myValue = "DAO Mock checkValue"
sut = ProductsUrlsRepo(dao=daoMock)
# Then
self.assertEqual("DAO Mock checkValue", sut._dao.myValue)
def __init__(self, shop: Shop, productsUrlsRepoPath: Path):
self.shop = shop
self.productsUrlsRepoPath = productsUrlsRepoPath
assert productsUrlsRepoPath.is_file()
productsUrlsDao = ProductsUrlsDao(productsUrlsRepoPath)
self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao)