def test_get_url(): scrape = Scraper() response = scrape.get_url( "https://domoplius.lt/skelbimai/butai?action_type=1&page_nr=1&slist=109410160" ) result_status = response.status_code assert result_status == 200
def add_product(category, url) -> None: print("Adding product...") logger = logging.getLogger(__name__) logger.info(f"Adding product: {category} - {url}") website_name = get_website_name(url) if website_name not in domains.keys(): print(f"Can't scrape from this website: {website_name}") logger.info("Not support website to scrape from") return new_product = Scraper(category, url) new_product.scrape_info() product_exists = check_if_product_exits(new_product) if not product_exists: add_product_to_records(new_product) Filemanager.add_product_to_csv(new_product.category, new_product.url) new_product.save_info() else: user_input = input( "A product with the same name and from the same website already exist in your data, do you want to override this product? (y/n) > " ) if user_input.lower() in ("y", "yes"): print("Overriding product...") add_product_to_records(new_product) Filemanager.add_product_to_csv(new_product.category, new_product.url) new_product.save_info() else: print("Product was not added nor overrided") logger.info("Adding product cancelled")
def test_scrape(mock_request): mock_request.get.return_value.status_code = 200 link_parser = mock.MagicMock() instance = link_parser() type(instance).next_url = mock.PropertyMock( side_effect=["http://a.b.com", None]) type(instance).links = mock.PropertyMock( side_effect=[["http://a", "http://b"], ["http://c"]]) listing_parser = mock.MagicMock() instance = listing_parser() type(instance).serialize = mock.PropertyMock( side_effect=["bloba", "blobb", "blobc"]) url = "http://this.url.is.unused" scraper = Scraper(start_url=url, link_parser=link_parser, listing_parser=listing_parser) scraper.run() assert Db.get("http://a") == "bloba" assert Db.get("http://b") == "blobb" assert Db.get("http://c") == "blobc"
def test_collect_information_price(): scrape = Scraper() data = scrape.collect_information(30) data1 = data['price'][0] assert data1 == '86 000 € (1 323 €/m²)'
def test_collect_information_floor(): scrape = Scraper() data = scrape.collect_information(30) data1 = data['floor'][0] assert data1 == '4/9'
def test_collect_information_year(): scrape = Scraper() data = scrape.collect_information(30) data1 = data['year'][0] assert data1 == '1980'
def test_collect_information_room(): scrape = Scraper() data = scrape.collect_information(30) data1 = data['room'][0] assert data1 == '3'
def test_collect_information_area(): scrape = Scraper() data = scrape.collect_information(30) data1 = data['area'][0] assert data1 == '65.00'
def test_collect_information_address(): scrape = Scraper() data = scrape.collect_information(30) data1 = data['address'][0] assert data1 == '3 kambarių butas Vilniuje, Šeškinėje, Dūkštų g. +15'
def test_get_page_number(): scrape = Scraper() result = scrape.get_page_number(31) assert result == 2