def __init__(self):
self.logfilePath = APP_USERDATA_DIR / "Logs/log_current.txt"
self.shopsRepoPath = APP_USERDATA_DIR / "Shops.json"
shopDao = TinyShopDao(path=self.shopsRepoPath)
self.productsUrlsRepoPath = APP_USERDATA_DIR / "ProductsURLs.txt"
productsUrlsDao = ProductsUrlsDao(filepath=self.productsUrlsRepoPath)
self.messengersRepoPath = APP_USERDATA_DIR / "Messengers.json"
discordMessengerDao = msn.DiscordTinyDao(path=self.messengersRepoPath)
self.proxiesRepoPath = APP_USERDATA_DIR / "Proxies.txt"
proxyDao = FileProxyDao(filepath=self.proxiesRepoPath)
self.userAgentsRepoPath = APP_USERDATA_DIR / "UserAgents.txt"
userAgentDao = FileUserAgentDao(filepath=self.userAgentsRepoPath)
self.shopRepo = ShopRepo(dao=shopDao)
self.productsUrlsRepo = ProductsUrlsRepo(dao=productsUrlsDao)
self.discordMessengerRepo = msn.Repo(dao=discordMessengerDao)
self.proxyRepo = ProxyRepo(dao=proxyDao)
self.userAgentRepo = UserAgentRepo(dao=userAgentDao)
self.session = None
self.scrapers: List[Scraper] = list()
self.shops: List[Shop] = list()
self._configureLogger()
self._createRepoFilesIfNotExist()
def test_addProxy(self):
# Given
testfileProxyDao = FileProxyDao(filepath=self.tempProxyRepoPath)
sut = ProxyRepo(dao=testfileProxyDao)
proxy1 = Proxy.make(endpoint="this.is.valid",
port=9384,
username="******",
pw="AndAPassword")
proxy2 = Proxy.make(endpoint="a.better.world.for.com",
port=2837,
username="******",
pw="GreatPassword")
# When
sut.addProxy(proxy1)
sut.addProxy(proxy2)
# Then
with open(str(self.tempProxyRepoPath), encoding="utf-8") as file:
lines = [line.rstrip("\n") for line in file.readlines()]
self.assertIsInstance(lines, list)
self.assertEqual(2, len(lines))
self.assertEqual("this.is.valid:9384:SomeUsername:AndAPassword",
lines[0])
self.assertEqual("a.better.world.for.com:2837:Myself:GreatPassword",
lines[1])
def test_init_shouldSetValidPath(self):
# Given
validPath = self.PATH_TO_6_VALID_PROXIES
# When
sut = FileProxyDao(validPath)
# Then
self.assertEqual(validPath, sut.connection.path)
def test_loadAll_shouldNotRaiseOnValidRepoFile(self):
with FileProxyDao(self.PATH_TO_6_VALID_PROXIES) as sut:
# When
try:
sut.loadAll()
# Then
except Exception as e:
self.fail(f"Expected no raise, but raised: {e}")
def test_loadAll_shouldFilterDisabledProxies(self):
# Given
path = Path(fixtures.network.TEST_6VALID_PROXIES_REPO_PATH)
# When
with FileProxyDao(path) as sut:
sut.loadAll()
# Then
self.assertEqual(6, len(sut._records.get(sut._recordArrayKey)))
def test_getRandomProxy_shouldReturnNoneIfRepoIsEmpty(self):
# Given
testfileProxyDao = FileProxyDao(filepath=TEST_0VALID_PROXIES_REPO_PATH)
sut = ProxyRepo(dao=testfileProxyDao)
# When
proxy = sut.getRandomProxy()
# Then
self.assertIsNone(proxy, f"Got value {proxy}")
def __init__(self):
self.proxyDao = FileProxyDao(
filepath=TEST_INTEGRATION_PROXIES_REPO_PATH)
self.proxyRepo = ProxyRepo(dao=self.proxyDao)
self.userAgentDao = FileUserAgentDao(
filepath=TEST_USERAGENTS_INTEGRATION_REPO_PATH)
self.userAgentRepo = UserAgentRepo(dao=self.userAgentDao)
self.messengerDao = msn.DiscordTinyDao(
path=TEST_MESSENGERS_INTEGRATION_REPO_PATH)
self.messengerRepo = msn.Repo(dao=self.messengerDao)
def test_loadAll_shouldFilterDuplicateProxies(self):
# Given
path = Path(fixtures.network.TEST_5WITH2DUPLICATES_PROXIES_REPO_PATH)
# When
with FileProxyDao(path) as sut:
sut.loadAll()
# Then
self.assertEqual(3, len(sut._records.get(sut._recordArrayKey)))
def test_contextManager(self):
# Given
path = fixtures.network.TEST_6VALID_PROXIES_REPO_PATH
# When
with FileProxyDao(path) as fileProxyDao:
proxies = fileProxyDao.loadAll()
# Then
self.assertIsInstance(proxies, list)
self.assertEqual(6, len(proxies))
def test_postprocess_shouldBeCalled(self):
path = Path(fixtures.network.TEST_EXACT3_PROXIES_REPO_PATH)
# When
with FileProxyDao(path) as sut:
with mock.patch("network.proxyDao.FileProxyDao._postprocess"
) as mockedMethod:
mockedMethod.return_value = ""
sut.loadAll()
mockedMethod.assert_called_once()
def test_loadAll_shouldNotRaiseIfFileIsNotEmptyButAllLinesAreInvalid(self):
# Given
path = Path(fixtures.network.TEST_0VALID_PROXIES_REPO_PATH)
with FileProxyDao(path) as sut:
# When
try:
sut.loadAll()
# Then
except Exception as e:
self.fail(f"Expected no raise, but raised: {e}")
def test_loadAll_shouldNotRaiseOnEmptyFile(self):
# Given
path = Path(fixtures.network.TEST_EMPTYFILE_PROXIES_REPO_PATH)
with FileProxyDao(path) as sut:
# When
try:
sut.loadAll()
# Then
except Exception as e:
self.fail(f"Expected no raise, but raised: {e}")
def test_filterRecord_shouldBeCalled(self):
path = Path(fixtures.network.TEST_EXACT3_PROXIES_REPO_PATH)
# When
with FileProxyDao(path) as sut:
with mock.patch("network.proxyDao.FileProxyDao._filterRecord"
) as mockedMethod:
mockedMethod.return_value = ""
sut.loadAll()
mockedMethod.assert_called()
self.assertEqual(3, mockedMethod.call_count)
def test_addProxy_shouldRaiseOnInvalidProxy(self):
# Given
testfileProxyDao = FileProxyDao(filepath=self.tempProxyRepoPath)
sut = ProxyRepo(dao=testfileProxyDao)
proxy1 = Proxy.make(
endpoint="Invalid endpoint, whitespace in string",
port=2938,
)
# When / Then
with self.assertRaises(ValueError):
sut.addProxy(proxy1)
def test_getRandomProxy(self):
# Given
testfileProxyDao = FileProxyDao(filepath=TEST_6VALID_PROXIES_REPO_PATH)
sut = ProxyRepo(dao=testfileProxyDao)
expectedMinimumDifferentProxies = 4
# When
returnedProxies = []
for i in range(1, 20):
proxy = sut.getRandomProxy()
if proxy not in returnedProxies:
returnedProxies.append(proxy)
if len(returnedProxies) == expectedMinimumDifferentProxies:
break
# Then
self.assertEqual(
expectedMinimumDifferentProxies, len(returnedProxies),
f"Expected minimum {expectedMinimumDifferentProxies} different proxies "
f"in 20 calls but got {len(returnedProxies)}")
def test_insert(self):
path = self.tempProxyRepoPath
# Write some existing valid proxies before testing insert.
existingProxies = [
"243.172.183.94:8344:creepy-user:creepy_pass\n",
"misoproponolpimpom:3344:jump-user:jump_pass"
]
with open(str(path), "w", encoding="utf-8") as file:
file.writelines(existingProxies)
# Data to be inserted
proxy1 = Proxy.make(endpoint="this.is.valid.com",
port=2938,
username="******",
pw="AndAPassword")
proxy2 = Proxy.make(endpoint="this.too.com",
port=8493,
username="******",
pw="GreatPassword")
with FileProxyDao(path) as sut:
# When
sut.insert(proxy1)
sut.insert(proxy2)
# Then
with open(str(path), encoding="utf-8") as file:
lines = [line.rstrip("\n") for line in file.readlines()]
self.assertIsInstance(lines, list)
self.assertEqual(4, len(lines))
self.assertEqual("243.172.183.94:8344:creepy-user:creepy_pass",
lines[0])
self.assertEqual("misoproponolpimpom:3344:jump-user:jump_pass",
lines[1])
self.assertEqual(
"this.is.valid.com:2938:SomeUsername:AndAPassword", lines[2])
self.assertEqual("this.too.com:8493:Myself:GreatPassword",
lines[3])
def test_insert_shouldStripDuplicates(self):
path = self.tempProxyRepoPath
# Write existing valid proxy before testing insert.
existingProxies = ["199.99.72.194:2837:creepy-user:creepy_pass\n"]
with open(str(path), "w", encoding="utf-8") as file:
file.writelines(existingProxies)
# Data to be inserted
proxy1 = Proxy.make(endpoint="this.is.valid.com",
port=2938,
username="******",
pw="AndAPassword")
proxy2 = Proxy.make(endpoint="this.too.com",
port=8493,
username="******",
pw="GreatPassword")
with FileProxyDao(path) as sut:
sut.insert(proxy1)
sut.insert(proxy2)
# When
sut.insert(
proxy1
) # proxy1 would be a duplicate should be stripped by DAO method
# Then
with open(str(path), "r", encoding="utf-8") as file:
allSaved = [line.rstrip("\n") for line in file.readlines()]
self.assertEqual(3, len(allSaved))
self.assertIn("199.99.72.194:2837:creepy-user:creepy_pass", allSaved)
self.assertIn("this.too.com:8493:Myself:GreatPassword", allSaved)
self.assertIn("this.is.valid.com:2938:SomeUsername:AndAPassword",
allSaved)
def __init__(self, dao: Dao = FileProxyDao()):
self._dao = dao
def test_init_shouldSetDefaultPathIfNoneIsGiven(self):
# When
sut = FileProxyDao()
# Then
self.assertIn(str(Path("/userdata/")), str(sut.connection.path))