def scrape_area(area):
"""
Scrapes Daft for a certain area, and finds the latest listings based on certain filtering criteria
:param area:
:return: A list of results.
"""
d = Daft()
offset = 0
pages = True
while pages:
daft_houses = d.get_listings(county=settings.COUNTY,
area=area,
offset=offset,
listing_type=settings.dwelling_type,
sale_type=settings.rent_or_sale,
max_price=settings.MAX_PRICE,
min_price=settings.MIN_PRICE)
results = []
if not daft_houses:
pages = False
for listing in daft_houses:
foo = session.query(Daft_Listing).filter_by(
link=listing.get_daft_link()).first()
# Don't store the listing if it already exists.
if foo is None:
# Create the listing object.
foo = Daft_Listing(link=listing.get_daft_link(),
created=listing.get_posted_since(),
building_type=listing.get_dwelling_type(),
address=listing.get_formalised_address(),
price=listing.get_price(),
bedrooms=listing.get_num_bedrooms(),
bathrooms=listing.get_num_bathrooms(),
area=listing.get_town())
# Save the listing so we don't grab it again.
session.add(foo)
session.commit()
price = listing.get_price()
if price is not None:
price = price.encode('utf-8')
results.append([
listing.get_formalised_address(), price,
listing.get_num_bedrooms(),
listing.get_daft_link()
])
offset += 10
return results
def test_read_xml(self):
daft = Daft()
daft.set_xml_url(
"http://daft.ie/rss.daft?uid=1685053&id=1106718&xk=858943")
listings = daft.read_xml()
self.assertGreater(len(listings), 0)
first = listings[0]
self.assertIsNotNone(first.formalised_address)
self.assertIsNotNone(first.price)
self.assertIsNotNone(first.daft_link)
def test_apartments_to_let(self):
daft = Daft()
daft.set_offset(20)
daft.set_listing_type(RentType.APARTMENTS)
daft.set_area("Malahide")
listings = daft.search(fetch_all=False)
search_count = daft.search_count
self.assertGreater(search_count, 0)
self.assertGreater(len(listings), 0)
apartment = listings[0]
print(apartment.daft_link)
self.assertIsNotNone(apartment.commercial_area_size)
self.assertIsNotNone(apartment.contact_number)
self.assertIsNotNone(apartment.daft_link)
self.assertIsNotNone(apartment.date_insert_update)
self.assertIsNotNone(apartment.facilities)
self.assertIsNotNone(apartment.formalised_address)
self.assertIsNotNone(apartment.id)
self.assertIsNotNone(apartment.bathrooms)
self.assertIsNotNone(apartment.bedrooms)
self.assertIsNotNone(apartment.overviews)
self.assertIsNotNone(apartment.price)
self.assertIsNotNone(apartment.search_type)
self.assertIsNotNone(apartment.shortcode)
self.assertIsNotNone(apartment.views)
self.assertIsNotNone(apartment.features)
self.assertIsNotNone(apartment.advertiser_name)
self.assertIsNotNone(apartment.date_insert_update)
self.assertIsNotNone(apartment.hires_images)
def test_area_commercial_properties(self):
daft = Daft()
daft.set_county("Dublin City")
daft.set_area("Dublin 15")
daft.set_listing_type(SaleType.COMMERCIAL)
listings = daft.get_listings()
self.assertTrue(len(listings) > 0)
def test_properties(self):
daft = Daft()
daft.set_county("Dublin City")
daft.set_area("Dublin 15")
daft.set_listing_type(SaleType.PROPERTIES)
listings = daft.get_listings()
self.assertTrue(len(listings) > 0)
def test_student_accommodation(self):
daft = Daft()
daft.set_listing_type(RentType.STUDENT_ACCOMMODATION)
daft.set_university(University.NCI)
daft.set_student_accommodation_type(
StudentAccommodationType.ROOMS_TO_SHARE)
listings = daft.search(fetch_all=False)
self.assertGreater(len(listings), 0)
def test_commercial_property_types(self):
daft = Daft()
daft.set_county("Dublin City")
daft.set_listing_type(SaleType.COMMERCIAL)
daft.set_commercial_property_type(CommercialType.OFFICE)
listings = daft.search()
self.assertTrue(len(listings) > 0)
def test_apartments_to_let(self):
daft = Daft()
daft.set_county("Dublin City")
daft.set_area("Dublin 15")
daft.set_listing_type(RentType.APARTMENTS)
listings = daft.get_listings()
self.assertTrue(len(listings) > 0)
def test_apartments_to_rent(self):
daft = Daft()
daft.set_search_type(SearchType.RESIDENTIAL_RENT)
daft.set_property_type(PropertyType.APARTMENT)
daft.set_location(Location.DUBLIN)
listings = daft.search(max_pages=1)
self.assertTrue(len(listings) > 0)
self.assertGreater(daft.total_results, 0)
def test_properties_with_max_beds(self):
daft = Daft()
daft.set_county('Dublin')
daft.set_min_beds(3)
daft.set_max_beds(3)
listings = daft.search()
self.assertTrue(len(listings) > 0)
listing = listings[0]
self.assertTrue(listing.bedrooms == 3)
def test_studios_to_rent(self):
daft = Daft()
daft.set_search_type(SearchType.RESIDENTIAL_RENT)
daft.set_property_type(PropertyType.STUDIO_APARTMENT)
daft.set_location(Location.DUBLIN)
listings = daft.search(max_pages=1)
self.assertTrue(len(listings) > 0)
self.assertTrue(listings[0].bedrooms == "1 bed")
self.assertGreater(daft.total_results, 0)
def test_search_multiple_areas(self, mock_post):
url = "https://search-gateway.dsch.ie/v1/listings"
payload = {
"section": "residential-to-rent",
"geoFilter": {
"storedShapeIds": ["2040", "2144", "2068"],
"geoSearchType": "STORED_SHAPES"
},
"paging": {
"from": "0",
"pagesize": "50"
},
}
headers = {
"Content-Type": "application/json",
"brand": "daft",
"platform": "web",
}
daft = Daft()
daft.set_search_type(SearchType.RESIDENTIAL_RENT)
daft.set_location(
[Location.ASHTOWN_DUBLIN, Location.IFSC_DUBLIN, "Blanchardstown"])
daft.search()
mock_post.assert_called_with(url, headers=headers, json=payload)
def test_open_viewing(self):
daft = Daft()
daft.set_open_viewing(True)
daft.set_listing_type(RentType.APARTMENTS)
listings = daft.search()
self.assertTrue(len(listings) > 0)
for listing in daft.search():
self.assertTrue(len(listing.upcoming_viewings) > 0)
def test_search_basic(self, mock_post):
url = "https://search-gateway.dsch.ie/v1/listings"
payload = {
"paging": {
"from": "0",
"pagesize": "50"
},
}
headers = {
"Content-Type": "application/json",
"brand": "daft",
"platform": "web",
}
daft = Daft()
daft.search()
mock_post.assert_called_with(url, headers=headers, json=payload)
def test_new_homes(self):
daft = Daft()
daft.set_search_type(SearchType.NEW_HOMES)
daft.set_location(Location.DUBLIN)
listings = daft.search(max_pages=1)
self.assertTrue(len(listings) > 0)
self.assertGreater(daft.total_results, 0)
def test_listing_type_exception(self):
daft = Daft()
daft.set_county("Meath")
has_raised_exception = False
try:
daft.set_listing_type("flat")
except Exception:
has_raised_exception = True
self.assertTrue(has_raised_exception)
def get_listings():
print('%s: Getting daft listings for %s' %
(time.strftime("%Y/%m/%d-%H:%M:%S"), RENT_TYPE))
daft = Daft()
# daft.set_county("Dublin City")
daft.set_county(LOCATION)
daft.set_listing_type(RENT_TYPE_MAP[RENT_TYPE])
# daft.set_min_price(0)
# daft.set_max_price(900)
# daft.set_address("Donnybrook")
# daft.set_address("Ballsbridge")
# daft.set_university(University.UCD)
offset = 0
all_listings = []
listings = daft.search()
all_listings.extend(listings)
# while True:
# daft.set_offset(offset)
# listings = daft.search()
# all_listings.extend(listings)
# if not listings:
# break
# offset += 20
print(len(all_listings))
# except GeocoderQuotaExceeded:
# geolocator = Nominatim(user_agent="specify_your_app_name_here")
# ucd_location = geolocator.geocode("O'Brien Centre for Science, Dublin")
# i = 0
# for listing in all_listings:
# address = listing.formalised_address
# print(i)
# listing.location = fix_address(address)
# if listing.location:
# listing.latitude = listing.location.latitude
# listing.longitude = listing.location.longitude
# listing.distance_to_ucd = get_distance(ucd_location, listing.location)
# else:
# listing.latitude = None
# listing.longitude = None
# listing.distance_to_ucd = None
# i += 1
return all_listings
def test_contact_advertiser(self):
daft = Daft()
daft.set_county("Meath")
daft.set_listing_type(RentType.FLAT)
listings = daft.search(fetch_all=False)
if len(listings) > 0:
first_listing = listings[0]
has_sent = first_listing.contact_advertiser(
name="Jane Doe",
contact_number="019202222",
email="*****@*****.**",
message=
"Hi, I seen your listing on daft.ie and I would like to schedule a viewing.",
)
self.assertTrue(has_sent)
# Get the current sale agreed prices for properties in Dublin 15 that are between 200,000 and 250,000.
from daftlistings import Daft, SaleType
daft = Daft()
daft.set_county("Dublin City")
daft.set_area("Dublin 15")
daft.set_listing_type(SaleType.PROPERTIES)
daft.set_sale_agreed(True)
daft.set_min_price(200000)
daft.set_max_price(250000)
listings = daft.get_listings()
for listing in listings:
print(listing.get_formalised_address())
print(listing.get_daft_link())
facilities = listing.get_facilities()
if facilities is not None:
print('Facilities: ')
for facility in facilities:
print(facility)
features = listing.get_features()
if features is not None:
print('Features: ')
for feature in features:
print(feature)
from daftlistings import Daft, RentType, Gender, SortOrder, SortType
daft = Daft()
areas = ['IFSC', 'Blackrock']
daft.set_county('Dublin City')
daft.set_area(areas)
daft.set_listing_type(RentType.ROOMS_TO_SHARE)
daft.set_with_photos(True)
daft.set_gender(Gender.MALE)
daft.set_verbose(True)
daft.set_sort_order(SortOrder.ASCENDING)
daft.set_sort_by(SortType.PRICE)
listings = daft.search()
for listing in listings:
print(listing.formalised_address)
print(listing.price)
print(listing.daft_link)
def test_apartments_to_let_with_price(self):
daft = Daft()
daft.set_county("Dublin City")
daft.set_area("Dublin 15")
daft.set_listing_type(RentType.APARTMENTS)
daft.set_min_price(1000)
daft.set_max_price(2000)
listings = daft.search(fetch_all=False)
self.assertTrue(len(listings) > 0)
listing = listings[0]
price = listing.price
self.assertTrue(1000 <= int(price) <= 2000)
def test_result_url(self):
daft = Daft()
daft.set_result_url("https://www.daft.ie/dublin/apartments-for-rent?")
listings = daft.search(fetch_all=False)
self.assertGreater(len(listings), 0)
def test_parking_spaces(self):
daft = Daft()
daft.set_listing_type(RentType.PARKING_SPACES)
listings = daft.search(fetch_all=False)
self.assertIsNotNone(listings)
def test_apartments_to_let(self):
daft = Daft()
daft.set_listing_type(RentType.APARTMENTS)
daft.set_area_type(AreaType.ENROUTE)
daft.set_public_transport_route(TransportRoute.BUS_LINE_15)
daft.set_min_lease(6)
daft.set_max_lease(12)
daft.set_offset(3)
daft.set_sort_order(SortOrder.DESCENDING)
daft.set_gender(Gender.EITHER)
daft.set_availability(12)
daft.set_pets_allowed(True)
daft.set_keywords("Furnished")
daft.set_couples_accepted(True)
listings = daft.search(fetch_all=False)
search_count = daft.search_count
self.assertGreater(search_count, 0)
self.assertGreater(len(listings), 0)
apartment = listings[0]
self.assertIsNotNone(apartment.commercial_area_size)
self.assertIsNotNone(apartment.contact_number)
self.assertIsNotNone(apartment.daft_link)
self.assertIsNotNone(apartment.date_insert_update)
self.assertIsNotNone(apartment.dwelling_type)
self.assertIsNotNone(apartment.facilities)
self.assertIsNotNone(apartment.formalised_address)
self.assertIsNotNone(apartment.id)
self.assertIsNotNone(apartment.bathrooms)
self.assertIsNotNone(apartment.bedrooms)
self.assertIsNotNone(apartment.overviews)
self.assertIsNotNone(apartment.price)
self.assertIsNotNone(apartment.search_type)
self.assertIsNotNone(apartment.shortcode)
self.assertIsNotNone(apartment.views)
self.assertIsNotNone(apartment.features)
self.assertIsNotNone(apartment.description)
self.assertIsNotNone(apartment.advertiser_name)
self.assertIsNotNone(apartment.agent)
self.assertIsNotNone(apartment.agent_url)
self.assertIsNotNone(apartment.ber_code)
self.assertIsNotNone(apartment.city_center_distance)
self.assertIsNotNone(apartment.date_insert_update)
self.assertIsNotNone(apartment.hires_images)
def test_lookup_via_address(self):
daft = Daft()
daft.set_address("Blackrock")
listings = daft.search(fetch_all=False)
self.assertTrue(len(listings) > 0)
self.assertTrue("Blackrock" in listings[0].formalised_address)
from daftlistings import Daft, SaleType, HouseType
daft = Daft()
daft.set_listing_type(SaleType.HOUSES)
daft.set_house_type(HouseType.DETACHED)
listings = daft.search(fetch_all=False)
for listing in listings:
print(listing.formalised_address)
print(listing.daft_link)
print(listing.price)
print(" ")
# Example to print a Listing object as dict.
from daftlistings import Daft, RentType, SortOrder, SortType
import pprint
daft = Daft(debug=True)
daft.set_county('Dublin')
daft.set_listing_type(RentType.APARTMENTS)
daft.set_sort_order(SortOrder.ASCENDING)
daft.set_sort_by(SortType.PRICE)
daft.set_with_photos(True)
daft.set_open_viewing(True)
listings = daft.search()
first = listings[0]
pprint.pprint(first.as_dict())
def test_room_to_share(self):
daft = Daft()
daft.set_county("Dublin")
daft.set_listing_type(RentType.ROOMS_TO_SHARE)
daft.set_with_photos(True)
daft.set_ensuite_only(True)
daft.set_furnished(True)
daft.set_num_occupants(2)
daft.set_min_price(500)
daft.set_max_price(1000)
daft.set_room_type(RoomType.DOUBLE)
listings = daft.search(fetch_all=False)
self.assertTrue(len(listings) > 0)
from daftlistings import Daft, Location, SearchType, SuitableFor
daft = Daft()
daft.set_location(Location.DUBLIN)
daft.set_search_type(SearchType.SHARING)
daft.set_owner_occupied(True)
daft.set_min_tenants(1)
daft.set_max_tenants(1)
daft.set_suitability(SuitableFor.MALE)
daft.set_min_price(1000)
daft.set_max_price(1000)
listings = daft.search()
for listing in listings:
print(listing.title)
print(listing.daft_link)
print("")
def test_as_dict(self):
daft = Daft()
daft.set_listing_type(RentType.APARTMENT_TO_SHARE)
listings = daft.search(fetch_all=False)
self.assertIsNotNone(listings[0].as_dict())
