def run(self) -> None:
driver = Driver()
driver = driver.get_driver()
driver.get(
'https://domclick.ru/search?offset=10&limit=10&sort_dir=desc')
html = driver.page_source
soup = BeautifulSoup(html)
max_ads = re.search(
'(\d*)',
soup.find('div', {
'class': 'listing-title'
}).text.replace(' ', '')).group(1)
for i in range(0, int(max_ads), 100):
soup = BeautifulSoup(
requests.get(self._ads_list_url_pattern.format(i)).text)
hrefs = list(map(lambda a: a.get('href', ''), soup.find_all('a')))
for href in hrefs:
if href.find('card') != -1:
self._data.put(href)
def get_page(self, url):
d = Driver().get_driver()
d.get(url)
return BeautifulSoup(d.page_source)
