def test_makeFromScrapee(self): # Given sut = ScraperFactory() class ScraperStub(Scraper): URL = "https://www.test_scraper_config_02.com" def __init__(self, scrapee, scrapeeRepo, request: Request, messenger): super().__init__(scrapee=scrapee, scrapeeRepo=scrapeeRepo, request=request, messenger=messenger) def run(self) -> None: raise NotImplementedError self.scraperStubClass = ScraperStub scrapee = Mock(spec_set=Scrapable) scrapeeRepo = Mock(spec_set=ShopRepo) session = Mock(spec=Session) messengerRequest = RequestMock() messenger = MessengerMock(request=messengerRequest) # Expecting values from ScraperConfigRepoMonkeyPatch repository for # scraper URL https://www.test_scraper_config_02.com expectedIterSleep = (7, 16, 1.0) expectedRequestTimeout = 5 expectedRequestMaxRetries = 5 expectedRequestUseRandomProxy = False scrapee.url = "https://www.test_scraper_config_02.com" scrapee.name = "The huge factory shop" sut.register(class_=self.scraperStubClass) # When createdScraper = sut.makeFromScrapee(scrapee=scrapee, scrapeeRepo=scrapeeRepo, requestClass=RequestMock, session=session, messenger=messenger) # Then self.assertIsInstance(createdScraper, self.scraperStubClass) self.assertIsInstance(createdScraper._scrapeeRepo, ShopRepo) self.assertIsInstance(createdScraper._request, Request) self.assertEqual(self.scraperStubClass.URL, createdScraper.URL) self.assertEqual(scrapee.name, createdScraper._scrapee.name) # Other attributes which are not initializable from outside self.assertEqual(False, createdScraper._isCancelLoop) self.assertEqual(0, createdScraper._failCount) self.assertEqual(expectedIterSleep, createdScraper._iterSleep) self.assertEqual(expectedRequestTimeout, createdScraper._request._timeout) self.assertEqual(expectedRequestMaxRetries, createdScraper._request._maxRetries) self.assertEqual(expectedRequestUseRandomProxy, createdScraper._request._useRandomProxy)
def test_makeFromScrapee_shouldRaiseOnScraperNotFound(self): # Given sut = ScraperFactory() nonFindableScrapee = Mock(spec_set=Scrapable) nonFindableScrapee.url = "https://non-findable-url.com" # When / Then with self.assertRaises(LookupError): sut.makeFromScrapee(scrapee=nonFindableScrapee, scrapeeRepo=Mock(), requestClass=Mock, session=Mock(), messenger=Mock())
def test_makeFromScrapees_shouldReturnEmptyListWhenNoScrapeesGiven(self): # Given sut = ScraperFactory() # When scrapers: List[Scraper] = sut.makeFromScrapees(scrapees=[], scrapeeRepo=Mock(), session=Mock(), requestClass=Mock, messenger=Mock()) # Then self.assertIsInstance(scrapers, list) self.assertEqual(0, len(scrapers))
def test_register_shouldNotRegisterClassesTwice(self): # Given sut = ScraperFactory() self.scraperStubClass = Mock(spec_set=Scraper) # When # Multiple registering expected to be ignored by register() sut.register(self.scraperStubClass) sut.register(self.scraperStubClass) # Then foundScraperMocks = list( filter(lambda typ: typ is self.scraperStubClass, sut._scraperClasses)) self.assertEqual(1, len(foundScraperMocks))
def test_makeFromScrapees_shouldReturnEmptyListWhenNoMatchingScrapers( self): # Given sut = ScraperFactory() scrapee = Mock(spec_set=Scrapable) scrapee.url = "https://should-lead-to-no-matching-scrapers.com" # When scrapers: List[Scraper] = sut.makeFromScrapees(scrapees=[scrapee], scrapeeRepo=Mock(), session=Mock(), requestClass=Mock, messenger=Mock()) # Then self.assertIsInstance(scrapers, list) self.assertEqual(0, len(scrapers))
async def _setScrapers(self): if not self.shops: raise AttributeError("Unable to create scrapers: Shops are not set.") if not self.session: raise AttributeError("Unable to create scrapers: Session not set.") messengerRequest: Request = AioHttpRequest(session=self.session) discordMessenger = msn.Discord(request=messengerRequest, repo=self.discordMessengerRepo) scraperFactory = ScraperFactory() self.scrapers = scraperFactory.makeFromScrapees( scrapees=self.shops, scrapeeRepo=self.shopRepo, session=self.session, requestClass=AioHttpRequest, messenger=discordMessenger) if not self.scrapers: raise LookupError("No scrapers were generated.")
def test_init_shouldSetExpectedValues(self): # When sut = ScraperFactory() # Then self.assertIsInstance(sut._scraperClasses, list) self.assertLessEqual(1, len(sut._scraperClasses)) for class_ in sut._scraperClasses: self.assertIsInstance(class_, type) self.assertTrue( issubclass(class_, Scraper), f"{class_} must be a subclass of Scraper but is not.")
def test_makeFromScrapees(self): # Given url1 = "https://scrapee-1.com" url2 = "https://scrapee-2.com" class Scraper1(Scraper): URL: str = url1 def run(self) -> None: pass class Scraper2(Scraper): URL: str = url2 def run(self) -> None: pass sut = ScraperFactory() sut.register(class_=Scraper1) sut.register(class_=Scraper2) scrapee1 = Mock(spec_set=Scrapable) scrapee1.url = url1 scrapee2 = Mock(spec_set=Scrapable) scrapee2.url = url2 scrapees = [scrapee1, scrapee2] # When scrapers: List[Scraper] = sut.makeFromScrapees(scrapees=scrapees, scrapeeRepo=Mock(), session=Mock(), requestClass=Mock, messenger=Mock()) # Then self.assertIsInstance(scrapers, list) self.assertEqual(2, len(scrapers))
def test_makeFromScrapee_shouldRaiseOnMultipleScrapersFound(self): # Given sut = ScraperFactory() scrapee = Mock(spec_set=Scrapable) scrapee.url.return_value = "does not matter here" class ScraperStub(Scraper): URL: str = "does not matter here" def run(self) -> None: pass self.scraperStubClass = ScraperStub # When sut._scraperClasses.append(self.scraperStubClass) sut._scraperClasses.append(self.scraperStubClass) with self.assertRaises(LookupError): sut.makeFromScrapee(scrapee=scrapee, scrapeeRepo=Mock(), session=Mock(), requestClass=Mock, messenger=Mock())
def test_register(self): # Given sut = ScraperFactory() shopScrapee = Mock(spec_set=Shop) shopRepo = Mock(spec_set=ShopRepo) request = Mock(spec_set=Request) shopScrapee.url = "https://my-supershop.url.com/" shopScrapee.name = "The Test Shop" # Then # Expect that there are one or more scraperClasses self.assertGreaterEqual(len(sut._scraperClasses), 1, "Expected at least 1 Scraper class.") # Expect that all registered scrapers are constrained to Interface 'Scrapable'. for scraperClass in sut._scraperClasses: self.assertTrue(issubclass(scraperClass, Scraper)) for scraperClass in sut._scraperClasses: try: # Expect that scraperClass is initializable # Initialize class: scraper = scraperClass(scrapee=shopScrapee, scrapeeRepo=shopRepo, request=request, messenger=Mock()) except Exception as e: self.fail( f"'scraper' object expected to be initializable, but raised: {e}" ) # Expect that scrapee and its URL were correctly passed to scraper self.assertIs(scraper._scrapee, shopScrapee) self.assertIs(scraper._scrapeeRepo, shopRepo) self.assertIs(scraper._request, request) self.assertEqual(shopScrapee.url, scraper._scrapee.url) self.assertEqual(shopScrapee.name, scraper._scrapee.name)