def test_findByName(self):
# Given
# Create test data to search for. We use two shops with the same name here.
shopData1 = dict(uid="ca0f5926-7d55-4973-a8e1-d3e2cc89fca6",
name="Shop with same name")
shopData2 = dict(uid="e68782fd-19af-428e-881f-99d7af9b83b0",
name="This shop should not be found")
shopData3 = dict(uid="b0e2e467-6fd5-4a06-bb1e-9ad60223cafa",
name="Shop with same name")
expectedShops = [Shop(**shopData1), Shop(**shopData3)]
with tdb.TinyDB(self.testDBPath) as db:
shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME)
shopTable.insert(shopData1)
shopTable.insert(shopData2)
shopTable.insert(shopData3)
# Setup repo
testTinyShopDao = TinyShopDao(path=self.testDBPath)
sut = ShopRepo(dao=testTinyShopDao)
# When
foundShops = sut.findByName("Shop with same name")
# Then
self.assertIsInstance(foundShops, list)
self.assertEqual(2, len(foundShops))
self.assertEqual(expectedShops, foundShops)
def __init__(self):
self.logfilePath = APP_USERDATA_DIR / "Logs/log_current.txt"
self.shopsRepoPath = APP_USERDATA_DIR / "Shops.json"
shopDao = TinyShopDao(path=self.shopsRepoPath)
self.productsUrlsRepoPath = APP_USERDATA_DIR / "ProductsURLs.txt"
productsUrlsDao = ProductsUrlsDao(filepath=self.productsUrlsRepoPath)
self.messengersRepoPath = APP_USERDATA_DIR / "Messengers.json"
discordMessengerDao = msn.DiscordTinyDao(path=self.messengersRepoPath)
self.proxiesRepoPath = APP_USERDATA_DIR / "Proxies.txt"
proxyDao = FileProxyDao(filepath=self.proxiesRepoPath)
self.userAgentsRepoPath = APP_USERDATA_DIR / "UserAgents.txt"
userAgentDao = FileUserAgentDao(filepath=self.userAgentsRepoPath)
self.shopRepo = ShopRepo(dao=shopDao)
self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao)
self.discordMessengerRepo = msn.Repo(dao=discordMessengerDao)
self.proxyRepo = ProxyRepo(dao=proxyDao)
self.userAgentRepo = UserAgentRepo(dao=userAgentDao)
self.session = None
self.scrapers: List[Scraper] = list()
self.shops: List[Shop] = list()
self._configureLogger()
self._createRepoFilesIfNotExist()
def test_findByUID(self):
# Given
# Create test data to search for.
uidToFind = "b0e2e467-6fd5-4a06-bb1e-9ad60223cafa"
shopData1 = dict(uid="ca0f5926-7d55-4973-a8e1-d3e2cc89fca6",
name="The name of the first test shop")
shopData2 = dict(uid=uidToFind,
name="The name of the second test shop")
expectedShop = Shop(**shopData2)
with tdb.TinyDB(self.testDBPath) as db:
shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME)
shopTable.insert(shopData1)
shopTable.insert(shopData2)
# Setup repo
testTinyShopDao = TinyShopDao(path=self.testDBPath)
sut = ShopRepo(dao=testTinyShopDao)
# When
foundShop = sut.findByUID(uidToFind)
# Then
self.assertIsInstance(foundShop, Shop)
self.assertEqual(foundShop.uid, uidToFind)
self.assertEqual(expectedShop, foundShop)
class ConcreteShopHelper:
""" Helper for shop scraping tests. Is able to create a fully scrapable
concrete shop. Able to create this shop as a record within a temporary shop repository. """
shop: Shop
shopDaoPath = TEMP_SHOPS_TINYDB_TEST_PATH
shopDao = TinyShopDao(path=shopDaoPath)
shopRepo = ShopRepo(dao=shopDao)
productsUrlsRepo: ProductsUrlsRepo
def __init__(self, shop: Shop, productsUrlsRepoPath: Path):
self.shop = shop
self.productsUrlsRepoPath = productsUrlsRepoPath
assert productsUrlsRepoPath.is_file()
productsUrlsDao = ProductsUrlsDao(productsUrlsRepoPath)
self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao)
def assignProductsFromProductsUrlsRepo(self):
# Generate shop products from products URLs (which are fixtures).
products = self.productsUrlsRepo.getAll()
assert len(products) > 0
if products:
self.shop.assignProducts(products)
assert len(self.shop.products) > 0
def overwriteRepoTableWithScrapableShop(self):
# Create fresh 'Shops' table with this shop as its data.
self.assignProductsFromProductsUrlsRepo()
self.shopRepo.setAll(shops=[self.shop])
def test_init_shouldSetDefaultValues(self):
# When
daoMock = Mock()
daoMock.myValue = "DAO Mock checkValue"
sut = ShopRepo(dao=daoMock)
# Then
self.assertEqual("DAO Mock checkValue", sut._dao.myValue)
def test_getAll(self):
# Given
testTinyShopDao = TinyShopDao(path=self.testDBPath)
# Create 2 shops in TinyDB for testing.
# Note that we use client code to create them, which is more of an integration test...
fixture = ShopFixture()
fixture.create2Shops()
expectedShops = fixture.shops
ShopRepo(dao=testTinyShopDao).setAll(shops=expectedShops)
sut = ShopRepo(dao=testTinyShopDao)
# When
loadedShops = sut.getAll()
# Then
# Expect that loaded shops match the expected
self.assertEqual(expectedShops, loadedShops)
def test_update(self):
# Given
# Create 2 shops in TinyDB for testing.
fixture = ShopFixture()
fixture.create2Shops()
expectedShop = fixture.shops[0]
assert expectedShop.uid is not None and expectedShop.uid != ""
# Write a shop which we can try to update by UID.
existingData = dict(uid=expectedShop.uid,
name="I don't know this shop's name")
with tdb.TinyDB(self.testDBPath) as db:
shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME)
shopTable.insert(existingData)
# Setup repo
testTinyShopDao = TinyShopDao(path=self.testDBPath)
sut = ShopRepo(dao=testTinyShopDao)
# When
sut.update(shop=expectedShop)
# Then
with tdb.TinyDB(self.testDBPath) as db:
shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME)
recordList: list = shopTable.all()
self.assertEqual(1, len(recordList))
# Expect that data with previous uid still exist
self.assertEqual(expectedShop.uid, recordList[0].get("uid"))
# Expect that shop's name has been updated
self.assertNotEqual("I don't know this shop's name",
recordList[0].get("name"))
# Note that we use client code to load the shop again, which is
# more of an integration test...
updatedShops = sut.getAll()
self.assertIsInstance(updatedShops, list)
self.assertEqual(1, len(recordList))
# Expect that updated shop matches the expected one
self.assertEqual(expectedShop, updatedShops[0])
class TempShopRepoHelper:
""" Helper for creating a controlling a fully scrapable temporary TinyDB shop repo.
The repository is created at initialization. It uses a path to a fixture file.
The shops table is overridden with data passed by the 'shops' argument at initialization. """
shopDaoPath: ClassVar[Path] = TEMP_SHOPS_TINYDB_TEST_PATH
shopDao: ClassVar[TinyShopDao] = TinyShopDao(path=shopDaoPath)
shopRepo: ClassVar[ShopRepo] = ShopRepo(dao=shopDao)
def __init__(self, shops: List[Shop]):
self.shops = shops
self.shopRepo.setAll(shops=self.shops)
def test_setAll(self):
# Given
# Insert a document into a fresh 'Shops' table. This data is expected to be
# completely overridden by the test.
existingData = dict(OneTestOne="Test data val 1",
TwoTestTwo="Test data val 2")
with tdb.TinyDB(self.testDBPath) as db:
shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME)
shopTable.insert(existingData)
# These data are expected:
fixture = ShopFixture()
fixture.create2Shops()
expectedShops = fixture.shops
# Setup repo
testTinyShopDao = TinyShopDao(path=self.testDBPath)
sut = ShopRepo(dao=testTinyShopDao)
# When
sut.setAll(shops=expectedShops)
# Then
with tdb.TinyDB(self.testDBPath) as db:
shopTable: tdb.database.Table = db.table(TinyShopDao._TABLE_NAME)
recordList: list = shopTable.all()
# Expect that previous data do not exist anymore
self.assertLessEqual(0, len(recordList))
self.assertIsNone(recordList[0].get("OneTestOne"))
self.assertIsNone(recordList[0].get("TwoTestTwo"))
# Note that we use client code to load the shops again, which is
# more of an integration test...
loadedShops = sut.getAll()
# Expect that loaded shops match the expected ones
self.assertEqual(expectedShops, loadedShops)
class Main:
def __init__(self):
self.logfilePath = APP_USERDATA_DIR / "Logs/log_current.txt"
self.shopsRepoPath = APP_USERDATA_DIR / "Shops.json"
shopDao = TinyShopDao(path=self.shopsRepoPath)
self.productsUrlsRepoPath = APP_USERDATA_DIR / "ProductsURLs.txt"
productsUrlsDao = ProductsUrlsDao(filepath=self.productsUrlsRepoPath)
self.messengersRepoPath = APP_USERDATA_DIR / "Messengers.json"
discordMessengerDao = msn.DiscordTinyDao(path=self.messengersRepoPath)
self.proxiesRepoPath = APP_USERDATA_DIR / "Proxies.txt"
proxyDao = FileProxyDao(filepath=self.proxiesRepoPath)
self.userAgentsRepoPath = APP_USERDATA_DIR / "UserAgents.txt"
userAgentDao = FileUserAgentDao(filepath=self.userAgentsRepoPath)
self.shopRepo = ShopRepo(dao=shopDao)
self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao)
self.discordMessengerRepo = msn.Repo(dao=discordMessengerDao)
self.proxyRepo = ProxyRepo(dao=proxyDao)
self.userAgentRepo = UserAgentRepo(dao=userAgentDao)
self.session = None
self.scrapers: List[Scraper] = list()
self.shops: List[Shop] = list()
self._configureLogger()
self._createRepoFilesIfNotExist()
async def run(self):
self._setShops()
try:
await self._startHttpSession()
await self._setScrapers()
# Create runners, start scraping
loopRunners = [s.loopRun() for s in self.scrapers]
await asyncio.gather(*loopRunners)
finally:
await self.session.close()
def _configureLogger(self):
loggerConfig = APP_CONFIG_REPO.findLoggerConfig()
clog.configureLogger(logger=clog.getRootLogger(),
config=loggerConfig,
logfilePath=self.logfilePath)
def _createRepoFilesIfNotExist(self):
if not self.shopsRepoPath.is_file():
self.shopsRepoPath.touch(exist_ok=False)
if not self.productsUrlsRepoPath.is_file():
self.productsUrlsRepoPath.touch(exist_ok=False)
if not self.messengersRepoPath.is_file():
self.messengersRepoPath.touch(exist_ok=False)
if not self.proxiesRepoPath.is_file():
self.proxiesRepoPath.touch(exist_ok=False)
if not self.userAgentsRepoPath.is_file():
self.userAgentsRepoPath.touch(exist_ok=False)
def _setShops(self):
# Update shop repo by looking through the ProductsUrls repository.
# This adds and/or deletes repo-shops and/or repo-products which do not correspond to
# the detected ProductsUrls.
self.shopRepo.updateFromProductsUrls(productsUrlsRepo=self.productsUrlsRepo)
try:
self.shops = self.shopRepo.getAll()
except Exception as e:
logger.warning("%s", e, exc_info=True)
raise
async def _startHttpSession(self):
self.session: Session = AioHttpSession(proxyRepo=self.proxyRepo,
userAgentRepo=self.userAgentRepo)
async def _setScrapers(self):
if not self.shops:
raise AttributeError("Unable to create scrapers: Shops are not set.")
if not self.session:
raise AttributeError("Unable to create scrapers: Session not set.")
messengerRequest: Request = AioHttpRequest(session=self.session)
discordMessenger = msn.Discord(request=messengerRequest, repo=self.discordMessengerRepo)
scraperFactory = ScraperFactory()
self.scrapers = scraperFactory.makeFromScrapees(
scrapees=self.shops,
scrapeeRepo=self.shopRepo,
session=self.session,
requestClass=AioHttpRequest,
messenger=discordMessenger)
if not self.scrapers:
raise LookupError("No scrapers were generated.")
def test_updateFromProductsUrls(self):
# Given
# Copy fixture to new arbitrary file as we will modify its contents within this test.
with open(str(PRODUCTS_URLS_9_VALID_TEST_PATH), "r",
encoding="utf-8") as source:
content = source.read()
with open(str(self.tempProductsUrlsRepoPath), "w+",
encoding="utf-8") as target:
target.write(content)
# Note that the table gets deleted by the unit test's setup() method - so we
# start with a fresh empty table.
testTinyShopDao = TinyShopDao(path=self.testDBPath)
sut = ShopRepo(dao=testTinyShopDao)
productsUrlsRepo = ProductsUrlsRepoMock(
productsUrlsRepoPath=self.tempProductsUrlsRepoPath)
expectedProducts = productsUrlsRepo.getAll()
expectedProductUrls = [p.url for p in expectedProducts]
# 1. Test initial update -----------------------------------------------------------
# When
# This is expected to fill the table with all the fixture data of ProductsUrls repo.
sut.updateFromProductsUrls(productsUrlsRepo=productsUrlsRepo)
# Then
shops = sut.getAll()
self.assertIsInstance(shops, list)
self.assertEqual(3, len(shops))
# Expect that all shops have been inserted
shopsUrls = [s.url for s in shops]
self.assertIn("https://www.solebox.com", shopsUrls)
self.assertIn("http://real.fantastic.de", shopsUrls)
self.assertIn("https://www.dbyte.org", shopsUrls)
# Expect that all products have been inserted
soleboxShop = list(
filter(lambda s: s.url == "https://www.solebox.com", shops))[0]
self.assertIsInstance(soleboxShop.products, list)
self.assertEqual(1, len(soleboxShop.products))
for product in soleboxShop.products:
self.assertIn(product.url, expectedProductUrls)
realFantasticShop = list(
filter(lambda s: s.url == "http://real.fantastic.de", shops))[0]
self.assertIsInstance(realFantasticShop.products, list)
self.assertEqual(2, len(realFantasticShop.products))
for product in realFantasticShop.products:
self.assertIn(product.url, expectedProductUrls)
dbyteShop = list(
filter(lambda s: s.url == "https://www.dbyte.org", shops))[0]
self.assertIsInstance(dbyteShop.products, list)
self.assertEqual(6, len(dbyteShop.products))
for product in dbyteShop.products:
self.assertIn(product.url, expectedProductUrls)
# 2. Test delete product/shop -----------------------------------------------------
# Given
# Remove all http://real.fantastic.de/... URLs from ProductsUrls repo.
with open(str(self.tempProductsUrlsRepoPath), "r+",
encoding="utf-8") as target:
lines = target.readlines()
for line in reversed(lines):
if line.startswith(
"http://real.fantastic.de/shop/great-realdumbtrump.htm"
):
lines.remove(line)
if line.startswith(
"http://real.fantastic.de/shop/buy-new-holo?prodid=682357ac"
):
lines.remove(line)
# Overwrite file with the updated data
target.seek(0)
target.writelines(lines)
# When
# This is expected to remove shop http://real.fantastic.de entirely from database,
# because it's products do not exist anymore in ProductsUrls repo.
sut.updateFromProductsUrls(productsUrlsRepo=productsUrlsRepo)
# Then
shops = sut.getAll()
self.assertIsInstance(shops, list)
self.assertEqual(2, len(shops))
# Expect that shop http://real.fantastic.de has been entirely removed from database
realFantasticShop = list(
filter(lambda s: s.url == "http://real.fantastic.de", shops))
self.assertIsInstance(realFantasticShop, list)
self.assertEqual(0, len(realFantasticShop))
# 3. Test add product to existing shop ----------------------------------------------
# Given
with open(str(self.tempProductsUrlsRepoPath), "r+",
encoding="utf-8") as target:
lines = target.readlines()
lines.append("\nhttps://www.solebox.com/some-new-product\n")
# Overwrite file with the updated data
target.seek(0)
target.writelines(lines)
expectedProducts = productsUrlsRepo.getAll()
expectedProductUrls = [p.url for p in expectedProducts]
# When
# This is expected to update shop https://www.solebox.com with the above added
# product https://www.solebox.com/some-new-product
sut.updateFromProductsUrls(productsUrlsRepo=productsUrlsRepo)
# Then
shops = sut.getAll()
self.assertIsInstance(shops, list)
self.assertEqual(2, len(shops))
# Expect that product https://www.solebox.com/some-new-product has been added to
# existing shop with URL https://www.solebox.com
soleboxShop = list(
filter(lambda s: s.url == "https://www.solebox.com", shops))[0]
self.assertIsInstance(soleboxShop.products, list)
self.assertEqual(2, len(soleboxShop.products))
for product in soleboxShop.products:
self.assertIn(product.url, expectedProductUrls)
# 4. Test add shop to existing shops -------------------------------------------------
# Given
with open(str(self.tempProductsUrlsRepoPath), "r+",
encoding="utf-8") as target:
lines = target.readlines()
lines.append("\nhttps://new-shop-1833663.com/new-product.htm\n")
# Overwrite file with the updated data
target.seek(0)
target.writelines(lines)
expectedProducts = productsUrlsRepo.getAll()
expectedProductUrls = [p.url for p in expectedProducts]
# When
# This is expected to update the shop table (which already has shops in it) with
# the above added product which has a base url which currently not exists
# in the shops table. So a new shop with this product must be created in shopRepo.
sut.updateFromProductsUrls(productsUrlsRepo=productsUrlsRepo)
# Then
shops = sut.getAll()
self.assertIsInstance(shops, list)
self.assertEqual(3, len(shops))
# Expect that shop https://new-shop-1833663.com has been added to
# existing database.
newShop = list(
filter(lambda s: s.url == "https://new-shop-1833663.com",
shops))[0]
self.assertIsInstance(newShop.products, list)
self.assertEqual(1, len(newShop.products))
for product in newShop.products:
self.assertIn(product.url, expectedProductUrls)