def test_makeFromScrapee(self): # Given sut = ScraperFactory() class ScraperStub(Scraper): URL = "https://www.test_scraper_config_02.com" def __init__(self, scrapee, scrapeeRepo, request: Request, messenger): super().__init__(scrapee=scrapee, scrapeeRepo=scrapeeRepo, request=request, messenger=messenger) def run(self) -> None: raise NotImplementedError self.scraperStubClass = ScraperStub scrapee = Mock(spec_set=Scrapable) scrapeeRepo = Mock(spec_set=ShopRepo) session = Mock(spec=Session) messengerRequest = RequestMock() messenger = MessengerMock(request=messengerRequest) # Expecting values from ScraperConfigRepoMonkeyPatch repository for # scraper URL https://www.test_scraper_config_02.com expectedIterSleep = (7, 16, 1.0) expectedRequestTimeout = 5 expectedRequestMaxRetries = 5 expectedRequestUseRandomProxy = False scrapee.url = "https://www.test_scraper_config_02.com" scrapee.name = "The huge factory shop" sut.register(class_=self.scraperStubClass) # When createdScraper = sut.makeFromScrapee(scrapee=scrapee, scrapeeRepo=scrapeeRepo, requestClass=RequestMock, session=session, messenger=messenger) # Then self.assertIsInstance(createdScraper, self.scraperStubClass) self.assertIsInstance(createdScraper._scrapeeRepo, ShopRepo) self.assertIsInstance(createdScraper._request, Request) self.assertEqual(self.scraperStubClass.URL, createdScraper.URL) self.assertEqual(scrapee.name, createdScraper._scrapee.name) # Other attributes which are not initializable from outside self.assertEqual(False, createdScraper._isCancelLoop) self.assertEqual(0, createdScraper._failCount) self.assertEqual(expectedIterSleep, createdScraper._iterSleep) self.assertEqual(expectedRequestTimeout, createdScraper._request._timeout) self.assertEqual(expectedRequestMaxRetries, createdScraper._request._maxRetries) self.assertEqual(expectedRequestUseRandomProxy, createdScraper._request._useRandomProxy)
def test_register_shouldNotRegisterClassesTwice(self): # Given sut = ScraperFactory() self.scraperStubClass = Mock(spec_set=Scraper) # When # Multiple registering expected to be ignored by register() sut.register(self.scraperStubClass) sut.register(self.scraperStubClass) # Then foundScraperMocks = list( filter(lambda typ: typ is self.scraperStubClass, sut._scraperClasses)) self.assertEqual(1, len(foundScraperMocks))
def test_makeFromScrapees(self): # Given url1 = "https://scrapee-1.com" url2 = "https://scrapee-2.com" class Scraper1(Scraper): URL: str = url1 def run(self) -> None: pass class Scraper2(Scraper): URL: str = url2 def run(self) -> None: pass sut = ScraperFactory() sut.register(class_=Scraper1) sut.register(class_=Scraper2) scrapee1 = Mock(spec_set=Scrapable) scrapee1.url = url1 scrapee2 = Mock(spec_set=Scrapable) scrapee2.url = url2 scrapees = [scrapee1, scrapee2] # When scrapers: List[Scraper] = sut.makeFromScrapees(scrapees=scrapees, scrapeeRepo=Mock(), session=Mock(), requestClass=Mock, messenger=Mock()) # Then self.assertIsInstance(scrapers, list) self.assertEqual(2, len(scrapers))