def test_makeFromScrapee(self):
# Given
sut = ScraperFactory()
class ScraperStub(Scraper):
URL = "https://www.test_scraper_config_02.com"
def __init__(self, scrapee, scrapeeRepo, request: Request,
messenger):
super().__init__(scrapee=scrapee,
scrapeeRepo=scrapeeRepo,
request=request,
messenger=messenger)
def run(self) -> None:
raise NotImplementedError
self.scraperStubClass = ScraperStub
scrapee = Mock(spec_set=Scrapable)
scrapeeRepo = Mock(spec_set=ShopRepo)
session = Mock(spec=Session)
messengerRequest = RequestMock()
messenger = MessengerMock(request=messengerRequest)
# Expecting values from ScraperConfigRepoMonkeyPatch repository for
# scraper URL https://www.test_scraper_config_02.com
expectedIterSleep = (7, 16, 1.0)
expectedRequestTimeout = 5
expectedRequestMaxRetries = 5
expectedRequestUseRandomProxy = False
scrapee.url = "https://www.test_scraper_config_02.com"
scrapee.name = "The huge factory shop"
sut.register(class_=self.scraperStubClass)
# When
createdScraper = sut.makeFromScrapee(scrapee=scrapee,
scrapeeRepo=scrapeeRepo,
requestClass=RequestMock,
session=session,
messenger=messenger)
# Then
self.assertIsInstance(createdScraper, self.scraperStubClass)
self.assertIsInstance(createdScraper._scrapeeRepo, ShopRepo)
self.assertIsInstance(createdScraper._request, Request)
self.assertEqual(self.scraperStubClass.URL, createdScraper.URL)
self.assertEqual(scrapee.name, createdScraper._scrapee.name)
# Other attributes which are not initializable from outside
self.assertEqual(False, createdScraper._isCancelLoop)
self.assertEqual(0, createdScraper._failCount)
self.assertEqual(expectedIterSleep, createdScraper._iterSleep)
self.assertEqual(expectedRequestTimeout,
createdScraper._request._timeout)
self.assertEqual(expectedRequestMaxRetries,
createdScraper._request._maxRetries)
self.assertEqual(expectedRequestUseRandomProxy,
createdScraper._request._useRandomProxy)
def test_makeFromScrapee_shouldRaiseOnScraperNotFound(self):
# Given
sut = ScraperFactory()
nonFindableScrapee = Mock(spec_set=Scrapable)
nonFindableScrapee.url = "https://non-findable-url.com"
# When / Then
with self.assertRaises(LookupError):
sut.makeFromScrapee(scrapee=nonFindableScrapee,
scrapeeRepo=Mock(),
requestClass=Mock,
session=Mock(),
messenger=Mock())
def test_makeFromScrapees_shouldReturnEmptyListWhenNoScrapeesGiven(self):
# Given
sut = ScraperFactory()
# When
scrapers: List[Scraper] = sut.makeFromScrapees(scrapees=[],
scrapeeRepo=Mock(),
session=Mock(),
requestClass=Mock,
messenger=Mock())
# Then
self.assertIsInstance(scrapers, list)
self.assertEqual(0, len(scrapers))
def test_register_shouldNotRegisterClassesTwice(self):
# Given
sut = ScraperFactory()
self.scraperStubClass = Mock(spec_set=Scraper)
# When
# Multiple registering expected to be ignored by register()
sut.register(self.scraperStubClass)
sut.register(self.scraperStubClass)
# Then
foundScraperMocks = list(
filter(lambda typ: typ is self.scraperStubClass,
sut._scraperClasses))
self.assertEqual(1, len(foundScraperMocks))
def test_makeFromScrapees_shouldReturnEmptyListWhenNoMatchingScrapers(
self):
# Given
sut = ScraperFactory()
scrapee = Mock(spec_set=Scrapable)
scrapee.url = "https://should-lead-to-no-matching-scrapers.com"
# When
scrapers: List[Scraper] = sut.makeFromScrapees(scrapees=[scrapee],
scrapeeRepo=Mock(),
session=Mock(),
requestClass=Mock,
messenger=Mock())
# Then
self.assertIsInstance(scrapers, list)
self.assertEqual(0, len(scrapers))
async def _setScrapers(self):
if not self.shops:
raise AttributeError("Unable to create scrapers: Shops are not set.")
if not self.session:
raise AttributeError("Unable to create scrapers: Session not set.")
messengerRequest: Request = AioHttpRequest(session=self.session)
discordMessenger = msn.Discord(request=messengerRequest, repo=self.discordMessengerRepo)
scraperFactory = ScraperFactory()
self.scrapers = scraperFactory.makeFromScrapees(
scrapees=self.shops,
scrapeeRepo=self.shopRepo,
session=self.session,
requestClass=AioHttpRequest,
messenger=discordMessenger)
if not self.scrapers:
raise LookupError("No scrapers were generated.")
def test_init_shouldSetExpectedValues(self):
# When
sut = ScraperFactory()
# Then
self.assertIsInstance(sut._scraperClasses, list)
self.assertLessEqual(1, len(sut._scraperClasses))
for class_ in sut._scraperClasses:
self.assertIsInstance(class_, type)
self.assertTrue(
issubclass(class_, Scraper),
f"{class_} must be a subclass of Scraper but is not.")
def test_makeFromScrapees(self):
# Given
url1 = "https://scrapee-1.com"
url2 = "https://scrapee-2.com"
class Scraper1(Scraper):
URL: str = url1
def run(self) -> None:
pass
class Scraper2(Scraper):
URL: str = url2
def run(self) -> None:
pass
sut = ScraperFactory()
sut.register(class_=Scraper1)
sut.register(class_=Scraper2)
scrapee1 = Mock(spec_set=Scrapable)
scrapee1.url = url1
scrapee2 = Mock(spec_set=Scrapable)
scrapee2.url = url2
scrapees = [scrapee1, scrapee2]
# When
scrapers: List[Scraper] = sut.makeFromScrapees(scrapees=scrapees,
scrapeeRepo=Mock(),
session=Mock(),
requestClass=Mock,
messenger=Mock())
# Then
self.assertIsInstance(scrapers, list)
self.assertEqual(2, len(scrapers))
def test_makeFromScrapee_shouldRaiseOnMultipleScrapersFound(self):
# Given
sut = ScraperFactory()
scrapee = Mock(spec_set=Scrapable)
scrapee.url.return_value = "does not matter here"
class ScraperStub(Scraper):
URL: str = "does not matter here"
def run(self) -> None:
pass
self.scraperStubClass = ScraperStub
# When
sut._scraperClasses.append(self.scraperStubClass)
sut._scraperClasses.append(self.scraperStubClass)
with self.assertRaises(LookupError):
sut.makeFromScrapee(scrapee=scrapee,
scrapeeRepo=Mock(),
session=Mock(),
requestClass=Mock,
messenger=Mock())
def test_register(self):
# Given
sut = ScraperFactory()
shopScrapee = Mock(spec_set=Shop)
shopRepo = Mock(spec_set=ShopRepo)
request = Mock(spec_set=Request)
shopScrapee.url = "https://my-supershop.url.com/"
shopScrapee.name = "The Test Shop"
# Then
# Expect that there are one or more scraperClasses
self.assertGreaterEqual(len(sut._scraperClasses), 1,
"Expected at least 1 Scraper class.")
# Expect that all registered scrapers are constrained to Interface 'Scrapable'.
for scraperClass in sut._scraperClasses:
self.assertTrue(issubclass(scraperClass, Scraper))
for scraperClass in sut._scraperClasses:
try:
# Expect that scraperClass is initializable
# Initialize class:
scraper = scraperClass(scrapee=shopScrapee,
scrapeeRepo=shopRepo,
request=request,
messenger=Mock())
except Exception as e:
self.fail(
f"'scraper' object expected to be initializable, but raised: {e}"
)
# Expect that scrapee and its URL were correctly passed to scraper
self.assertIs(scraper._scrapee, shopScrapee)
self.assertIs(scraper._scrapeeRepo, shopRepo)
self.assertIs(scraper._request, request)
self.assertEqual(shopScrapee.url, scraper._scrapee.url)
self.assertEqual(shopScrapee.name, scraper._scrapee.name)