def scrape_area(area): """ Scrapes Daft for a certain area, and finds the latest listings based on certain filtering criteria :param area: :return: A list of results. """ d = Daft() offset = 0 pages = True while pages: daft_houses = d.get_listings(county=settings.COUNTY, area=area, offset=offset, listing_type=settings.dwelling_type, sale_type=settings.rent_or_sale, max_price=settings.MAX_PRICE, min_price=settings.MIN_PRICE) results = [] if not daft_houses: pages = False for listing in daft_houses: foo = session.query(Daft_Listing).filter_by( link=listing.get_daft_link()).first() # Don't store the listing if it already exists. if foo is None: # Create the listing object. foo = Daft_Listing(link=listing.get_daft_link(), created=listing.get_posted_since(), building_type=listing.get_dwelling_type(), address=listing.get_formalised_address(), price=listing.get_price(), bedrooms=listing.get_num_bedrooms(), bathrooms=listing.get_num_bathrooms(), area=listing.get_town()) # Save the listing so we don't grab it again. session.add(foo) session.commit() price = listing.get_price() if price is not None: price = price.encode('utf-8') results.append([ listing.get_formalised_address(), price, listing.get_num_bedrooms(), listing.get_daft_link() ]) offset += 10 return results
def test_read_xml(self): daft = Daft() daft.set_xml_url( "http://daft.ie/rss.daft?uid=1685053&id=1106718&xk=858943") listings = daft.read_xml() self.assertGreater(len(listings), 0) first = listings[0] self.assertIsNotNone(first.formalised_address) self.assertIsNotNone(first.price) self.assertIsNotNone(first.daft_link)
def test_apartments_to_let(self): daft = Daft() daft.set_offset(20) daft.set_listing_type(RentType.APARTMENTS) daft.set_area("Malahide") listings = daft.search(fetch_all=False) search_count = daft.search_count self.assertGreater(search_count, 0) self.assertGreater(len(listings), 0) apartment = listings[0] print(apartment.daft_link) self.assertIsNotNone(apartment.commercial_area_size) self.assertIsNotNone(apartment.contact_number) self.assertIsNotNone(apartment.daft_link) self.assertIsNotNone(apartment.date_insert_update) self.assertIsNotNone(apartment.facilities) self.assertIsNotNone(apartment.formalised_address) self.assertIsNotNone(apartment.id) self.assertIsNotNone(apartment.bathrooms) self.assertIsNotNone(apartment.bedrooms) self.assertIsNotNone(apartment.overviews) self.assertIsNotNone(apartment.price) self.assertIsNotNone(apartment.search_type) self.assertIsNotNone(apartment.shortcode) self.assertIsNotNone(apartment.views) self.assertIsNotNone(apartment.features) self.assertIsNotNone(apartment.advertiser_name) self.assertIsNotNone(apartment.date_insert_update) self.assertIsNotNone(apartment.hires_images)
def test_area_commercial_properties(self): daft = Daft() daft.set_county("Dublin City") daft.set_area("Dublin 15") daft.set_listing_type(SaleType.COMMERCIAL) listings = daft.get_listings() self.assertTrue(len(listings) > 0)
def test_properties(self): daft = Daft() daft.set_county("Dublin City") daft.set_area("Dublin 15") daft.set_listing_type(SaleType.PROPERTIES) listings = daft.get_listings() self.assertTrue(len(listings) > 0)
def test_student_accommodation(self): daft = Daft() daft.set_listing_type(RentType.STUDENT_ACCOMMODATION) daft.set_university(University.NCI) daft.set_student_accommodation_type( StudentAccommodationType.ROOMS_TO_SHARE) listings = daft.search(fetch_all=False) self.assertGreater(len(listings), 0)
def test_commercial_property_types(self): daft = Daft() daft.set_county("Dublin City") daft.set_listing_type(SaleType.COMMERCIAL) daft.set_commercial_property_type(CommercialType.OFFICE) listings = daft.search() self.assertTrue(len(listings) > 0)
def test_apartments_to_let(self): daft = Daft() daft.set_county("Dublin City") daft.set_area("Dublin 15") daft.set_listing_type(RentType.APARTMENTS) listings = daft.get_listings() self.assertTrue(len(listings) > 0)
def test_apartments_to_rent(self): daft = Daft() daft.set_search_type(SearchType.RESIDENTIAL_RENT) daft.set_property_type(PropertyType.APARTMENT) daft.set_location(Location.DUBLIN) listings = daft.search(max_pages=1) self.assertTrue(len(listings) > 0) self.assertGreater(daft.total_results, 0)
def test_properties_with_max_beds(self): daft = Daft() daft.set_county('Dublin') daft.set_min_beds(3) daft.set_max_beds(3) listings = daft.search() self.assertTrue(len(listings) > 0) listing = listings[0] self.assertTrue(listing.bedrooms == 3)
def test_studios_to_rent(self): daft = Daft() daft.set_search_type(SearchType.RESIDENTIAL_RENT) daft.set_property_type(PropertyType.STUDIO_APARTMENT) daft.set_location(Location.DUBLIN) listings = daft.search(max_pages=1) self.assertTrue(len(listings) > 0) self.assertTrue(listings[0].bedrooms == "1 bed") self.assertGreater(daft.total_results, 0)
def test_search_multiple_areas(self, mock_post): url = "https://search-gateway.dsch.ie/v1/listings" payload = { "section": "residential-to-rent", "geoFilter": { "storedShapeIds": ["2040", "2144", "2068"], "geoSearchType": "STORED_SHAPES" }, "paging": { "from": "0", "pagesize": "50" }, } headers = { "Content-Type": "application/json", "brand": "daft", "platform": "web", } daft = Daft() daft.set_search_type(SearchType.RESIDENTIAL_RENT) daft.set_location( [Location.ASHTOWN_DUBLIN, Location.IFSC_DUBLIN, "Blanchardstown"]) daft.search() mock_post.assert_called_with(url, headers=headers, json=payload)
def test_open_viewing(self): daft = Daft() daft.set_open_viewing(True) daft.set_listing_type(RentType.APARTMENTS) listings = daft.search() self.assertTrue(len(listings) > 0) for listing in daft.search(): self.assertTrue(len(listing.upcoming_viewings) > 0)
def test_search_basic(self, mock_post): url = "https://search-gateway.dsch.ie/v1/listings" payload = { "paging": { "from": "0", "pagesize": "50" }, } headers = { "Content-Type": "application/json", "brand": "daft", "platform": "web", } daft = Daft() daft.search() mock_post.assert_called_with(url, headers=headers, json=payload)
def test_new_homes(self): daft = Daft() daft.set_search_type(SearchType.NEW_HOMES) daft.set_location(Location.DUBLIN) listings = daft.search(max_pages=1) self.assertTrue(len(listings) > 0) self.assertGreater(daft.total_results, 0)
def test_listing_type_exception(self): daft = Daft() daft.set_county("Meath") has_raised_exception = False try: daft.set_listing_type("flat") except Exception: has_raised_exception = True self.assertTrue(has_raised_exception)
def get_listings(): print('%s: Getting daft listings for %s' % (time.strftime("%Y/%m/%d-%H:%M:%S"), RENT_TYPE)) daft = Daft() # daft.set_county("Dublin City") daft.set_county(LOCATION) daft.set_listing_type(RENT_TYPE_MAP[RENT_TYPE]) # daft.set_min_price(0) # daft.set_max_price(900) # daft.set_address("Donnybrook") # daft.set_address("Ballsbridge") # daft.set_university(University.UCD) offset = 0 all_listings = [] listings = daft.search() all_listings.extend(listings) # while True: # daft.set_offset(offset) # listings = daft.search() # all_listings.extend(listings) # if not listings: # break # offset += 20 print(len(all_listings)) # except GeocoderQuotaExceeded: # geolocator = Nominatim(user_agent="specify_your_app_name_here") # ucd_location = geolocator.geocode("O'Brien Centre for Science, Dublin") # i = 0 # for listing in all_listings: # address = listing.formalised_address # print(i) # listing.location = fix_address(address) # if listing.location: # listing.latitude = listing.location.latitude # listing.longitude = listing.location.longitude # listing.distance_to_ucd = get_distance(ucd_location, listing.location) # else: # listing.latitude = None # listing.longitude = None # listing.distance_to_ucd = None # i += 1 return all_listings
def test_contact_advertiser(self): daft = Daft() daft.set_county("Meath") daft.set_listing_type(RentType.FLAT) listings = daft.search(fetch_all=False) if len(listings) > 0: first_listing = listings[0] has_sent = first_listing.contact_advertiser( name="Jane Doe", contact_number="019202222", email="*****@*****.**", message= "Hi, I seen your listing on daft.ie and I would like to schedule a viewing.", ) self.assertTrue(has_sent)
# Get the current sale agreed prices for properties in Dublin 15 that are between 200,000 and 250,000. from daftlistings import Daft, SaleType daft = Daft() daft.set_county("Dublin City") daft.set_area("Dublin 15") daft.set_listing_type(SaleType.PROPERTIES) daft.set_sale_agreed(True) daft.set_min_price(200000) daft.set_max_price(250000) listings = daft.get_listings() for listing in listings: print(listing.get_formalised_address()) print(listing.get_daft_link()) facilities = listing.get_facilities() if facilities is not None: print('Facilities: ') for facility in facilities: print(facility) features = listing.get_features() if features is not None: print('Features: ') for feature in features: print(feature)
from daftlistings import Daft, RentType, Gender, SortOrder, SortType daft = Daft() areas = ['IFSC', 'Blackrock'] daft.set_county('Dublin City') daft.set_area(areas) daft.set_listing_type(RentType.ROOMS_TO_SHARE) daft.set_with_photos(True) daft.set_gender(Gender.MALE) daft.set_verbose(True) daft.set_sort_order(SortOrder.ASCENDING) daft.set_sort_by(SortType.PRICE) listings = daft.search() for listing in listings: print(listing.formalised_address) print(listing.price) print(listing.daft_link)
def test_apartments_to_let_with_price(self): daft = Daft() daft.set_county("Dublin City") daft.set_area("Dublin 15") daft.set_listing_type(RentType.APARTMENTS) daft.set_min_price(1000) daft.set_max_price(2000) listings = daft.search(fetch_all=False) self.assertTrue(len(listings) > 0) listing = listings[0] price = listing.price self.assertTrue(1000 <= int(price) <= 2000)
def test_result_url(self): daft = Daft() daft.set_result_url("https://www.daft.ie/dublin/apartments-for-rent?") listings = daft.search(fetch_all=False) self.assertGreater(len(listings), 0)
def test_parking_spaces(self): daft = Daft() daft.set_listing_type(RentType.PARKING_SPACES) listings = daft.search(fetch_all=False) self.assertIsNotNone(listings)
def test_apartments_to_let(self): daft = Daft() daft.set_listing_type(RentType.APARTMENTS) daft.set_area_type(AreaType.ENROUTE) daft.set_public_transport_route(TransportRoute.BUS_LINE_15) daft.set_min_lease(6) daft.set_max_lease(12) daft.set_offset(3) daft.set_sort_order(SortOrder.DESCENDING) daft.set_gender(Gender.EITHER) daft.set_availability(12) daft.set_pets_allowed(True) daft.set_keywords("Furnished") daft.set_couples_accepted(True) listings = daft.search(fetch_all=False) search_count = daft.search_count self.assertGreater(search_count, 0) self.assertGreater(len(listings), 0) apartment = listings[0] self.assertIsNotNone(apartment.commercial_area_size) self.assertIsNotNone(apartment.contact_number) self.assertIsNotNone(apartment.daft_link) self.assertIsNotNone(apartment.date_insert_update) self.assertIsNotNone(apartment.dwelling_type) self.assertIsNotNone(apartment.facilities) self.assertIsNotNone(apartment.formalised_address) self.assertIsNotNone(apartment.id) self.assertIsNotNone(apartment.bathrooms) self.assertIsNotNone(apartment.bedrooms) self.assertIsNotNone(apartment.overviews) self.assertIsNotNone(apartment.price) self.assertIsNotNone(apartment.search_type) self.assertIsNotNone(apartment.shortcode) self.assertIsNotNone(apartment.views) self.assertIsNotNone(apartment.features) self.assertIsNotNone(apartment.description) self.assertIsNotNone(apartment.advertiser_name) self.assertIsNotNone(apartment.agent) self.assertIsNotNone(apartment.agent_url) self.assertIsNotNone(apartment.ber_code) self.assertIsNotNone(apartment.city_center_distance) self.assertIsNotNone(apartment.date_insert_update) self.assertIsNotNone(apartment.hires_images)
def test_lookup_via_address(self): daft = Daft() daft.set_address("Blackrock") listings = daft.search(fetch_all=False) self.assertTrue(len(listings) > 0) self.assertTrue("Blackrock" in listings[0].formalised_address)
from daftlistings import Daft, SaleType, HouseType daft = Daft() daft.set_listing_type(SaleType.HOUSES) daft.set_house_type(HouseType.DETACHED) listings = daft.search(fetch_all=False) for listing in listings: print(listing.formalised_address) print(listing.daft_link) print(listing.price) print(" ")
# Example to print a Listing object as dict. from daftlistings import Daft, RentType, SortOrder, SortType import pprint daft = Daft(debug=True) daft.set_county('Dublin') daft.set_listing_type(RentType.APARTMENTS) daft.set_sort_order(SortOrder.ASCENDING) daft.set_sort_by(SortType.PRICE) daft.set_with_photos(True) daft.set_open_viewing(True) listings = daft.search() first = listings[0] pprint.pprint(first.as_dict())
def test_room_to_share(self): daft = Daft() daft.set_county("Dublin") daft.set_listing_type(RentType.ROOMS_TO_SHARE) daft.set_with_photos(True) daft.set_ensuite_only(True) daft.set_furnished(True) daft.set_num_occupants(2) daft.set_min_price(500) daft.set_max_price(1000) daft.set_room_type(RoomType.DOUBLE) listings = daft.search(fetch_all=False) self.assertTrue(len(listings) > 0)
from daftlistings import Daft, Location, SearchType, SuitableFor daft = Daft() daft.set_location(Location.DUBLIN) daft.set_search_type(SearchType.SHARING) daft.set_owner_occupied(True) daft.set_min_tenants(1) daft.set_max_tenants(1) daft.set_suitability(SuitableFor.MALE) daft.set_min_price(1000) daft.set_max_price(1000) listings = daft.search() for listing in listings: print(listing.title) print(listing.daft_link) print("")
def test_as_dict(self): daft = Daft() daft.set_listing_type(RentType.APARTMENT_TO_SHARE) listings = daft.search(fetch_all=False) self.assertIsNotNone(listings[0].as_dict())