def test_addProxy(self):
# Given
testfileProxyDao = FileProxyDao(filepath=self.tempProxyRepoPath)
sut = ProxyRepo(dao=testfileProxyDao)
proxy1 = Proxy.make(endpoint="this.is.valid",
port=9384,
username="******",
pw="AndAPassword")
proxy2 = Proxy.make(endpoint="a.better.world.for.com",
port=2837,
username="******",
pw="GreatPassword")
# When
sut.addProxy(proxy1)
sut.addProxy(proxy2)
# Then
with open(str(self.tempProxyRepoPath), encoding="utf-8") as file:
lines = [line.rstrip("\n") for line in file.readlines()]
self.assertIsInstance(lines, list)
self.assertEqual(2, len(lines))
self.assertEqual("this.is.valid:9384:SomeUsername:AndAPassword",
lines[0])
self.assertEqual("a.better.world.for.com:2837:Myself:GreatPassword",
lines[1])
def test_getRandomProxy_shouldReturnNoneIfRepoIsEmpty(self):
# Given
testfileProxyDao = FileProxyDao(filepath=TEST_0VALID_PROXIES_REPO_PATH)
sut = ProxyRepo(dao=testfileProxyDao)
# When
proxy = sut.getRandomProxy()
# Then
self.assertIsNone(proxy, f"Got value {proxy}")
def test_addProxy_shouldRaiseOnInvalidProxy(self):
# Given
testfileProxyDao = FileProxyDao(filepath=self.tempProxyRepoPath)
sut = ProxyRepo(dao=testfileProxyDao)
proxy1 = Proxy.make(
endpoint="Invalid endpoint, whitespace in string",
port=2938,
)
# When / Then
with self.assertRaises(ValueError):
sut.addProxy(proxy1)
def __init__(self):
self.logfilePath = APP_USERDATA_DIR / "Logs/log_current.txt"
self.shopsRepoPath = APP_USERDATA_DIR / "Shops.json"
shopDao = TinyShopDao(path=self.shopsRepoPath)
self.productsUrlsRepoPath = APP_USERDATA_DIR / "ProductsURLs.txt"
productsUrlsDao = ProductsUrlsDao(filepath=self.productsUrlsRepoPath)
self.messengersRepoPath = APP_USERDATA_DIR / "Messengers.json"
discordMessengerDao = msn.DiscordTinyDao(path=self.messengersRepoPath)
self.proxiesRepoPath = APP_USERDATA_DIR / "Proxies.txt"
proxyDao = FileProxyDao(filepath=self.proxiesRepoPath)
self.userAgentsRepoPath = APP_USERDATA_DIR / "UserAgents.txt"
userAgentDao = FileUserAgentDao(filepath=self.userAgentsRepoPath)
self.shopRepo = ShopRepo(dao=shopDao)
self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao)
self.discordMessengerRepo = msn.Repo(dao=discordMessengerDao)
self.proxyRepo = ProxyRepo(dao=proxyDao)
self.userAgentRepo = UserAgentRepo(dao=userAgentDao)
self.session = None
self.scrapers: List[Scraper] = list()
self.shops: List[Shop] = list()
self._configureLogger()
self._createRepoFilesIfNotExist()
def __init__(self,
proxyRepo: ProxyRepo = ProxyRepo(),
userAgentRepo: UserAgentRepo = UserAgentRepo(),
*args,
**kwargs):
super().__init__(*args, **kwargs)
Session.__init__(self, proxyRepo, userAgentRepo)
logger.debug("AioHttpSession initialized")
def test_init_shouldSetDefaultValues(self):
# When
daoMock = Mock()
daoMock.myValue = "DAO Mock checkValue"
sut = ProxyRepo(dao=daoMock)
# Then
self.assertEqual("DAO Mock checkValue", sut._dao.myValue)
def __init__(self):
self.proxyDao = FileProxyDao(
filepath=TEST_INTEGRATION_PROXIES_REPO_PATH)
self.proxyRepo = ProxyRepo(dao=self.proxyDao)
self.userAgentDao = FileUserAgentDao(
filepath=TEST_USERAGENTS_INTEGRATION_REPO_PATH)
self.userAgentRepo = UserAgentRepo(dao=self.userAgentDao)
self.messengerDao = msn.DiscordTinyDao(
path=TEST_MESSENGERS_INTEGRATION_REPO_PATH)
self.messengerRepo = msn.Repo(dao=self.messengerDao)
def test_getRandomProxy(self):
# Given
testfileProxyDao = FileProxyDao(filepath=TEST_6VALID_PROXIES_REPO_PATH)
sut = ProxyRepo(dao=testfileProxyDao)
expectedMinimumDifferentProxies = 4
# When
returnedProxies = []
for i in range(1, 20):
proxy = sut.getRandomProxy()
if proxy not in returnedProxies:
returnedProxies.append(proxy)
if len(returnedProxies) == expectedMinimumDifferentProxies:
break
# Then
self.assertEqual(
expectedMinimumDifferentProxies, len(returnedProxies),
f"Expected minimum {expectedMinimumDifferentProxies} different proxies "
f"in 20 calls but got {len(returnedProxies)}")