def test_makeFromScrapee(self):
# Given
sut = ScraperFactory()
class ScraperStub(Scraper):
URL = "https://www.test_scraper_config_02.com"
def __init__(self, scrapee, scrapeeRepo, request: Request,
messenger):
super().__init__(scrapee=scrapee,
scrapeeRepo=scrapeeRepo,
request=request,
messenger=messenger)
def run(self) -> None:
raise NotImplementedError
self.scraperStubClass = ScraperStub
scrapee = Mock(spec_set=Scrapable)
scrapeeRepo = Mock(spec_set=ShopRepo)
session = Mock(spec=Session)
messengerRequest = RequestMock()
messenger = MessengerMock(request=messengerRequest)
# Expecting values from ScraperConfigRepoMonkeyPatch repository for
# scraper URL https://www.test_scraper_config_02.com
expectedIterSleep = (7, 16, 1.0)
expectedRequestTimeout = 5
expectedRequestMaxRetries = 5
expectedRequestUseRandomProxy = False
scrapee.url = "https://www.test_scraper_config_02.com"
scrapee.name = "The huge factory shop"
sut.register(class_=self.scraperStubClass)
# When
createdScraper = sut.makeFromScrapee(scrapee=scrapee,
scrapeeRepo=scrapeeRepo,
requestClass=RequestMock,
session=session,
messenger=messenger)
# Then
self.assertIsInstance(createdScraper, self.scraperStubClass)
self.assertIsInstance(createdScraper._scrapeeRepo, ShopRepo)
self.assertIsInstance(createdScraper._request, Request)
self.assertEqual(self.scraperStubClass.URL, createdScraper.URL)
self.assertEqual(scrapee.name, createdScraper._scrapee.name)
# Other attributes which are not initializable from outside
self.assertEqual(False, createdScraper._isCancelLoop)
self.assertEqual(0, createdScraper._failCount)
self.assertEqual(expectedIterSleep, createdScraper._iterSleep)
self.assertEqual(expectedRequestTimeout,
createdScraper._request._timeout)
self.assertEqual(expectedRequestMaxRetries,
createdScraper._request._maxRetries)
self.assertEqual(expectedRequestUseRandomProxy,
createdScraper._request._useRandomProxy)
def test_register_shouldNotRegisterClassesTwice(self):
# Given
sut = ScraperFactory()
self.scraperStubClass = Mock(spec_set=Scraper)
# When
# Multiple registering expected to be ignored by register()
sut.register(self.scraperStubClass)
sut.register(self.scraperStubClass)
# Then
foundScraperMocks = list(
filter(lambda typ: typ is self.scraperStubClass,
sut._scraperClasses))
self.assertEqual(1, len(foundScraperMocks))
def test_makeFromScrapees(self):
# Given
url1 = "https://scrapee-1.com"
url2 = "https://scrapee-2.com"
class Scraper1(Scraper):
URL: str = url1
def run(self) -> None:
pass
class Scraper2(Scraper):
URL: str = url2
def run(self) -> None:
pass
sut = ScraperFactory()
sut.register(class_=Scraper1)
sut.register(class_=Scraper2)
scrapee1 = Mock(spec_set=Scrapable)
scrapee1.url = url1
scrapee2 = Mock(spec_set=Scrapable)
scrapee2.url = url2
scrapees = [scrapee1, scrapee2]
# When
scrapers: List[Scraper] = sut.makeFromScrapees(scrapees=scrapees,
scrapeeRepo=Mock(),
session=Mock(),
requestClass=Mock,
messenger=Mock())
# Then
self.assertIsInstance(scrapers, list)
self.assertEqual(2, len(scrapers))