def test_get_url():
scrape = Scraper()
response = scrape.get_url(
"https://domoplius.lt/skelbimai/butai?action_type=1&page_nr=1&slist=109410160"
)
result_status = response.status_code
assert result_status == 200
def add_product(category, url) -> None:
print("Adding product...")
logger = logging.getLogger(__name__)
logger.info(f"Adding product: {category} - {url}")
website_name = get_website_name(url)
if website_name not in domains.keys():
print(f"Can't scrape from this website: {website_name}")
logger.info("Not support website to scrape from")
return
new_product = Scraper(category, url)
new_product.scrape_info()
product_exists = check_if_product_exits(new_product)
if not product_exists:
add_product_to_records(new_product)
Filemanager.add_product_to_csv(new_product.category, new_product.url)
new_product.save_info()
else:
user_input = input(
"A product with the same name and from the same website already exist in your data, do you want to override this product? (y/n) > "
)
if user_input.lower() in ("y", "yes"):
print("Overriding product...")
add_product_to_records(new_product)
Filemanager.add_product_to_csv(new_product.category,
new_product.url)
new_product.save_info()
else:
print("Product was not added nor overrided")
logger.info("Adding product cancelled")
def test_scrape(mock_request):
mock_request.get.return_value.status_code = 200
link_parser = mock.MagicMock()
instance = link_parser()
type(instance).next_url = mock.PropertyMock(
side_effect=["http://a.b.com", None])
type(instance).links = mock.PropertyMock(
side_effect=[["http://a", "http://b"], ["http://c"]])
listing_parser = mock.MagicMock()
instance = listing_parser()
type(instance).serialize = mock.PropertyMock(
side_effect=["bloba", "blobb", "blobc"])
url = "http://this.url.is.unused"
scraper = Scraper(start_url=url,
link_parser=link_parser,
listing_parser=listing_parser)
scraper.run()
assert Db.get("http://a") == "bloba"
assert Db.get("http://b") == "blobb"
assert Db.get("http://c") == "blobc"
def test_collect_information_price():
scrape = Scraper()
data = scrape.collect_information(30)
data1 = data['price'][0]
assert data1 == '86 000 € (1 323 €/m²)'
def test_collect_information_floor():
scrape = Scraper()
data = scrape.collect_information(30)
data1 = data['floor'][0]
assert data1 == '4/9'
def test_collect_information_year():
scrape = Scraper()
data = scrape.collect_information(30)
data1 = data['year'][0]
assert data1 == '1980'
def test_collect_information_room():
scrape = Scraper()
data = scrape.collect_information(30)
data1 = data['room'][0]
assert data1 == '3'
def test_collect_information_area():
scrape = Scraper()
data = scrape.collect_information(30)
data1 = data['area'][0]
assert data1 == '65.00'
def test_collect_information_address():
scrape = Scraper()
data = scrape.collect_information(30)
data1 = data['address'][0]
assert data1 == '3 kambarių butas Vilniuje, Šeškinėje, Dūkštų g. +15'
def test_get_page_number():
scrape = Scraper()
result = scrape.get_page_number(31)
assert result == 2