def main(htmlstring, driver): table_name = "maricopa_30_08_2020" header = { 'accept': '*/*', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'en-US,en;q=0.9,ko;q=0.8', 'cookie' : 'zguid=23|%2410ab80e6-80db-4e0a-9f70-2449ca972d74; _ga=GA1.2.759159145.1599348167; zjs_user_id=null; zjs_anonymous_id=%2210ab80e6-80db-4e0a-9f70-2449ca972d74%22; _gcl_au=1.1.607943717.1599348169; _pxvid=be9ff2f0-efce-11ea-9652-0242ac12000b; __gads=ID=cab593cad6cbce43:T=1599348200:S=ALNI_MaFYrYCZZvPIITKUEoEDXGvXSRYwQ; _gid=GA1.2.1287304564.1599556314; _pin_unauth=dWlkPU9EUXdZamxrTldJdE9ESTBNUzAwWXprMExXSXdNekl0TkdWak0yWTFNVEE1TldJeSZycD1abUZzYzJV; ki_r=; ki_s=; _fbp=fb.1.1599562363584.1440832488; g_state={"i_p":1599570378147,"i_l":1}; ki_t=1599556892885%3B1599556892885%3B1599563330503%3B1%3B19; JSESSIONID=62F47C1DAFBF00B3DB7B301BEA3E6586; zgsession=1|8840c1ee-f8a6-43d7-9a7b-3169df33c987; _pxff_cc=U2FtZVNpdGU9TGF4Ow==; _pxff_rf=1; _pxff_fp=1; _pxff_bsco=1; _px3=6d722620cec81d0df86c8eff4b631bdd93cef163fb0a14808e80f81013747454:M7trNae6CpAztMArZT97P3Vy9jFLz9FuEZ5p2efYpXeqOJC7Bw+xzsVGxArAYe+PM+vQKNuEI3qytjutx2UEXg==:1000:M1Vo/kdU1lI8Zqky6jJnuwSu45xHxX8ueCLKUiW6KX8rNR+VWAORLQi+1ns4dhilOU7gSCJfJmToj1SeyKN49kHZQZIQ0wSFeFtn+txzkIo/fhFAr2Cq7WvjCVWw7GBx8F3JIjMqHf1BZAAFg0YXqy/IVuCFhvIioSyK35nkm4A=; _gat=1; KruxPixel=true; DoubleClickSession=true; _uetsid=f44fc66ca5c392a6859170ed776b6ae9; _uetvid=dc708dafb2b6d91ab6c6923ac1ae6673; AWSALB=3gLhoP6QCdmf4zskymQ7ej/kbqzRHNkv+QNQMFmS6Y7S9pENaOusdnQVhFHWm1W9z8/1Og/WmO8JK63ys0wmi6ZNwRc4SN8lf4pcoyrm+nj8lLAPLRDIqMaYAEte; AWSALBCORS=3gLhoP6QCdmf4zskymQ7ej/kbqzRHNkv+QNQMFmS6Y7S9pENaOusdnQVhFHWm1W9z8/1Og/WmO8JK63ys0wmi6ZNwRc4SN8lf4pcoyrm+nj8lLAPLRDIqMaYAEte; search=6|1602203173818%7Crb%3DMaricopa%252C-AZ%26rect%3D33.203401%252C-111.882231%252C32.788612%252C-112.512953%26disp%3Dmap%26mdm%3Dauto%26sort%3Ddays%26pt%3Dpmf%252Cpf%26fs%3D1%26fr%3D0%26rs%3D0%26ah%3D0%26singlestory%3D0%26abo%3D0%26garage%3D0%26pool%3D0%26ac%3D0%26waterfront%3D0%26finished%3D0%26unfinished%3D0%26cityview%3D0%26mountainview%3D0%26parkview%3D0%26waterview%3D0%26hoadata%3D1%26zillow-owned%3D0%263dhome%3D0%09%0932697%09%09%09%09%09%09', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36' } first_url = "https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState=%7B%22pagination%22%3A%7B%7D%2C%22usersSearchTerm%22%3A%2285302%22%2C%22mapBounds%22%3A%7B%22west%22%3A-112.2113015161133%2C%22east%22%3A-112.14744348388673%2C%22south%22%3A33.53905815305113%2C%22north%22%3A33.59870248455097%7D%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A94884%2C%22regionType%22%3A7%7D%5D%2C%22isMapVisible%22%3Atrue%2C%22mapZoom%22%3A14%2C%22filterState%22%3A%7B%22sort%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%7D%2C%22isListVisible%22%3Atrue%7D&includeMap=false&includeList=true" default_url = "https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState=%7B%22pagination%22%3A%7B%22currentPage%22%3A{}%7D%2C%22usersSearchTerm%22%3A%2285302%22%2C%22mapBounds%22%3A%7B%22west%22%3A-112.2113015161133%2C%22east%22%3A-112.14744348388673%2C%22south%22%3A33.53905815305113%2C%22north%22%3A33.59870248455097%7D%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A94884%2C%22regionType%22%3A7%7D%5D%2C%22isMapVisible%22%3Atrue%2C%22mapZoom%22%3A14%2C%22filterState%22%3A%7B%22sort%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%7D%2C%22isListVisible%22%3Atrue%7D&includeMap=false&includeList=true" counts = 1 for page in range(1, 3): if page == 1: url = first_url else: url = default_url.format(page) response = requests.get(url, headers=header) result = response.json() properties_infos = result["searchResults"]["listResults"] print(len(properties_infos)) for i in range(0, len(properties_infos)): data_base = [] property_url = properties_infos[i]["detailUrl"] status_text = properties_infos[i]["statusText"] print(status_text, counts) counts += 1 try: street_add = properties_infos[i]["hdpData"]["homeInfo"]["streetAddress"] except: street_add = "" try: city = properties_infos[i]["hdpData"]["homeInfo"]["city"] except: city = "" try: state = properties_infos[i]["hdpData"]["homeInfo"]["state"] except: state = "" try: zipcode = properties_infos[i]["hdpData"]["homeInfo"]["zipcode"] except: zipcode = "" property_address = street_add + ", " + city + ", " + state + " " + zipcode try: bathrooms = properties_infos[i]["hdpData"]["homeInfo"]["bathrooms"] except: bathrooms = "" try: bedrooms = properties_infos[i]["hdpData"]["homeInfo"]["bedrooms"] except: bedrooms = "" try: tax_assessed_value = properties_infos[i]["hdpData"]["homeInfo"]["taxAssessedValue"] except: tax_assessed_value = "" try: zestimate = properties_infos[i]["hdpData"]["homeInfo"]["zestimate"] except: zestimate = "" try: rent_zestimate = properties_infos[i]["hdpData"]["homeInfo"]["rentZestimate"] except: rent_zestimate = "" try: home_type = properties_infos[i]["hdpData"]["homeInfo"]["homeType"] except: home_type = "" if "by owner" in status_text: print("--------------------------------------------------> : ", i + 1) driver.get(property_url) time.sleep(10) try: wait(driver, "//ul[@class='ds-home-fact-list']") except: print("There is no xpath") # street_add = driver.find_element_by_xpath("//h1[@class='ds-address-container']/span[1]").text # property_address = street_add + ", " + city + ", " + state + " " + zipcode # phone_number = driver.find_element_by_xpath("//span[@class='listing-field']").text phones = re.findall(r'[(][\d]{3}[)][ ]?[\d]{3}-[\d]{4}', driver.page_source) for phone in range(1, len(phones) + 1): phone_number = phones[phone - 1] features_labels = driver.find_elements_by_xpath("//ul[@class='ds-home-fact-list']//span[contains(@class, 'ds-standard-label') and contains(@class, 'ds-home-fact-label')]") features_infos = driver.find_elements_by_xpath("//ul[@class='ds-home-fact-list']//span[contains(@class, 'ds-body') and contains(@class, 'ds-home-fact-value')]") parking = "" year_built = "" hoa = "" heating = "" lot = "" cooling = "" price_sqft = "" for feature_label, feature_info in zip(features_labels, features_infos): feature_label_txt = feature_label.text if 'Parking' in feature_label_txt: parking = feature_info.text elif 'Year built' in feature_label_txt: year_built = feature_info.text elif 'HOA' in feature_label_txt: hoa = feature_info.text elif 'Heating' in feature_label_txt: heating = feature_info.text elif 'Lot' in feature_label_txt: lot = feature_info.text elif 'Cooling' in feature_label_txt: cooling = feature_info.text elif 'Price/' in feature_label_txt: price_sqft = feature_info.text print("Property Address--------------------> : ", property_address) print("Property Url------------------------> : ", property_url) print("Property Status---------------------> : ", status_text) print("Owner Phone Number------------------> : ", phone_number) print("BathRooms---------------------------> : ", bathrooms) print("BedRooms----------------------------> : ", bedrooms) print("Tax Assessed Value------------------> : ", tax_assessed_value) print("Zestimate---------------------------> : ", zestimate) print("Rent Zestimate----------------------> : ", rent_zestimate) print("Home Type---------------------------> : ", home_type) print("Parking-----------------------------> : ", parking) print("Year Built--------------------------> : ", year_built) print("HOA---------------------------------> : ", hoa) print("Heating-----------------------------> : ", heating) print("Lot---------------------------------> : ", lot) print("Cooling-----------------------------> : ", cooling) print("Price Sqft--------------------------> : ", price_sqft) string_id = property_address + status_text + phone_number m = hashlib.md5() m.update(string_id.encode('utf8')) identifier = m.hexdigest() print("hash-------------------->", identifier) create_time = str(datetime.datetime.now()) update_time = "" insertdb = InsertDB() data_base.append((property_address, street_add, city, state, zipcode, status_text, phone_number, bathrooms, bedrooms, tax_assessed_value, zestimate, rent_zestimate, home_type, parking, year_built, hoa, heating, lot, cooling, price_sqft, identifier, create_time, update_time)) insertdb.insert_document(data_base, table_name)
def main(htmlstring, driver): table_name = "maricopa" header = { 'accept': '*/*', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'en-US,en;q=0.9,ko;q=0.8', 'referer': 'https://www.zillow.com/homes/85139_rb/', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36' } pagination = "" usersSearchTerm = "85263" west = "-111.7398158178711" east = "-111.63063918212892" south = "33.683945743230794" north = "33.802992604350585" regionId = "94855" regionType = "7" mapZoom = "13" includeList = "true" # https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={%22pagination%22:{},%22usersSearchTerm%22:%2285006%22,%22mapBounds%22:{%22west%22:-112.07973577801513,%22east%22:-112.01665022198486,%22south%22:33.43522122804253,%22north%22:33.494937169247144},%22regionSelection%22:[{%22regionId%22:94722,%22regionType%22:7}],%22isMapVisible%22:true,%22mapZoom%22:14,%22filterState%22:{%22sort%22:{%22value%22:%22globalrelevanceex%22}},%22isListVisible%22:true}&includeMap=false&includeList=true default_first_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{0},"usersSearchTerm":"{1}","mapBounds":{"west":{2},"east":{3},"south":{4},"north":{5}},"regionSelection":[{"regionId":{6},"regionType":{7}}],"isMapVisible":true,"mapZoom":{8},"filterState":{"sort":{"value":"globalrelevanceex"}},"isListVisible":true}&includeMap=false&includeList={9}' first_case_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{' + pagination + '},' + '"usersSearchTerm":"' + usersSearchTerm + '","mapBounds":{"west":' + west + ',"east":' + east + ',"south":' + south + ',"north":' + north + '},"regionSelection":[{"regionId":' + regionId + ',"regionType":' + regionType + '}],"isMapVisible":true,"mapZoom":' + mapZoom + ',"filterState":{"sort":{"value":"globalrelevanceex"}},"isListVisible":true}&includeMap=false&includeList=' + includeList # first_url = default_first_url.format(pagination, usersSearchTerm, west, east, south, north, regionId, regionType, mapZoom, includeList) print(first_case_url) # return default_page_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{"currentPage":' + pagination + '},' + '"usersSearchTerm":"' + usersSearchTerm + '","mapBounds":{"west":' + west + ',"east":' + east + ',"south":' + south + ',"north":' + north + '},"regionSelection":[{"regionId":' + regionId + ',"regionType":' + regionType + '}],"isMapVisible":true,"mapZoom":' + mapZoom + ',"filterState":{"sort":{"value":"globalrelevanceex"}},"isListVisible":true}&includeMap=false&includeList=' + includeList counts = 1 for page in range(1, 6): default_page_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{"currentPage":' + str(page) + '},' + '"usersSearchTerm":"' + usersSearchTerm + '","mapBounds":{"west":' + west + ',"east":' + east + ',"south":' + south + ',"north":' + north + '},"regionSelection":[{"regionId":' + regionId + ',"regionType":' + regionType + '}],"isMapVisible":true,"mapZoom":' + mapZoom + ',"filterState":{"sort":{"value":"globalrelevanceex"}},"isListVisible":true}&includeMap=false&includeList=' + includeList if page == 1: url = first_case_url else: url = default_page_url response = requests.get(url, headers=header) result = response.json() properties_infos = result["searchResults"]["listResults"] print(len(properties_infos)) for i in range(0, len(properties_infos)): data_base = [] property_url = properties_infos[i]["detailUrl"] status_text = properties_infos[i]["statusText"] print(status_text, counts) counts += 1 try: street_add = properties_infos[i]["hdpData"]["homeInfo"]["streetAddress"] except: street_add = "" try: city = properties_infos[i]["hdpData"]["homeInfo"]["city"] except: city = "" try: state = properties_infos[i]["hdpData"]["homeInfo"]["state"] except: state = "" try: zipcode = properties_infos[i]["hdpData"]["homeInfo"]["zipcode"] except: zipcode = "" property_address = street_add + ", " + city + ", " + state + " " + zipcode if "by owner" in status_text: print("--------------------------------------------------> : ", i + 1) driver.get(property_url) time.sleep(10) # street_add = driver.find_element_by_xpath("//h1[@class='ds-address-container']/span[1]").text # property_address = street_add + ", " + city + ", " + state + " " + zipcode # phone_number = driver.find_element_by_xpath("//span[@class='listing-field']").text phones = re.findall(r'[(][\d]{3}[)][ ]?[\d]{3}-[\d]{4}', driver.page_source) for phone in range(1, len(phones) + 1): phone_number = phones[phone - 1] print("Property Address--------------------> : ", property_address) print("Property Url------------------------> : ", property_url) print("Property Status---------------------> : ", status_text) print("Owner Phone Number------------------> : ", phone_number) string_id = property_address + status_text + phone_number m = hashlib.md5() m.update(string_id.encode('utf8')) identifier = m.hexdigest() print("hash-------------------->", identifier) create_time = str(datetime.datetime.now()) update_time = "" insertdb = InsertDB() data_base.append((property_address, street_add, city, state, zipcode, status_text, phone_number, identifier, create_time, update_time)) insertdb.insert_document(data_base, table_name)
def main(htmlstring, driver): table_name = "maricopa" header = { 'accept': '*/*', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'en-US,en;q=0.9,ko;q=0.8', 'referer': 'https://www.zillow.com/homes/85139_rb/', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36' } first_url = "https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState=%7B%22pagination%22%3A%7B%7D%2C%22usersSearchTerm%22%3A%2285326%22%2C%22mapBounds%22%3A%7B%22west%22%3A-113.0331127578125%2C%22east%22%3A-112.0113842421875%2C%22south%22%3A32.70202923402303%2C%22north%22%3A33.66057991855808%7D%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A94903%2C%22regionType%22%3A7%7D%5D%2C%22isMapVisible%22%3Atrue%2C%22filterState%22%3A%7B%22sort%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%7D%2C%22isListVisible%22%3Atrue%7D&includeMap=false&includeList=true" default_url = "https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState=%7B%22pagination%22%3A%7B%22currentPage%22%3A{}%7D%2C%22usersSearchTerm%22%3A%2285326%22%2C%22mapBounds%22%3A%7B%22west%22%3A-113.0331127578125%2C%22east%22%3A-112.0113842421875%2C%22south%22%3A32.70202923402303%2C%22north%22%3A33.66057991855808%7D%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A94903%2C%22regionType%22%3A7%7D%5D%2C%22isMapVisible%22%3Atrue%2C%22filterState%22%3A%7B%22sort%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%7D%2C%22isListVisible%22%3Atrue%7D&includeMap=false&includeList=true" for page in range(1, 10): if page == 1: url = first_url else: url = default_url.format(page) response = requests.get(url, headers=header) result = response.json() properties_infos = result["searchResults"]["listResults"] print(len(properties_infos)) for i in range(0, len(properties_infos)): data_base = [] property_url = properties_infos[i]["detailUrl"] status_text = properties_infos[i]["statusText"] print(status_text) try: street_add = properties_infos[i]["hdpData"]["homeInfo"]["streetAddress"] except: street_add = "" try: city = properties_infos[i]["hdpData"]["homeInfo"]["city"] except: city = "" try: state = properties_infos[i]["hdpData"]["homeInfo"]["state"] except: state = "" try: zipcode = properties_infos[i]["hdpData"]["homeInfo"]["zipcode"] except: zipcode = "" property_address = street_add + ", " + city + ", " + state + " " + zipcode if "by owner" in status_text: print("--------------------------------------------------> : ", i + 1) print("Property Address--------------------> : ", property_address) print("Property Url------------------------> : ", property_url) print("Property Status---------------------> : ", status_text) driver.get(property_url) time.sleep(10) # phone_number = driver.find_element_by_xpath("//span[@class='listing-field']").text phones = re.findall(r'[(][\d]{3}[)][ ]?[\d]{3}-[\d]{4}', driver.page_source) for phone in range(1, len(phones) + 1): phone_number = phones[phone - 1] print("Owner Phone Number------------------> : ", phone_number) string_id = property_address + status_text + phone_number m = hashlib.md5() m.update(string_id.encode('utf8')) identifier = m.hexdigest() print("hash-------------------->", identifier) create_time = str(datetime.datetime.now()) update_time = "" insertdb = InsertDB() data_base.append((property_address, street_add, city, state, zipcode, status_text, phone_number, identifier, create_time, update_time)) insertdb.insert_document(data_base, table_name)
def main(htmlstring, driver): table_name = "maricopa_30_08_2020" header = { 'accept': '*/*', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'en-US,en;q=0.9,ko;q=0.8', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36' } pagination = "" usersSearchTerm = "Maricopa County, AZ" west = "-114.00266022265627" east = "-110.50900787890627" south = "32.012669442967976" north = "35.813297084142235" regionId = "2402" regionType = "4" mapZoom = "8" includeList = "true" priceMax = "150000" priceMin = "130000" monthlyPayMax = "511" monthlyPayMin = "443" # https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={%22pagination%22:{},%22usersSearchTerm%22:%2285006%22,%22mapBounds%22:{%22west%22:-112.07973577801513,%22east%22:-112.01665022198486,%22south%22:33.43522122804253,%22north%22:33.494937169247144},%22regionSelection%22:[{%22regionId%22:94722,%22regionType%22:7}],%22isMapVisible%22:true,%22mapZoom%22:14,%22filterState%22:{%22sort%22:{%22value%22:%22globalrelevanceex%22}},%22isListVisible%22:true}&includeMap=false&includeList=true default_first_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{0},"usersSearchTerm":"{1}","mapBounds":{"west":{2},"east":{3},"south":{4},"north":{5}},"regionSelection":[{"regionId":{6},"regionType":{7}}],"isMapVisible":true,"mapZoom":{8},"filterState":{"sort":{"value":"globalrelevanceex"}},"isListVisible":true}&includeMap=false&includeList={9}' first_case_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{' + pagination + '},' + '"usersSearchTerm":"' + usersSearchTerm + '","mapBounds":{"west":' + west + ',"east":' + east + ',"south":' + south + ',"north":' + north + '},"regionSelection":[{"regionId":' + regionId + ',"regionType":' + regionType + '}],"isMapVisible":true,"mapZoom":' + mapZoom + ',"filterState":{"sort":{"value":"globalrelevanceex"},"price":{"max":' + priceMax + ',"min":' + priceMin + '},"monthlyPayment":{"max":' + monthlyPayMax + ',"min":' + monthlyPayMin + '}},"isListVisible":true}&includeMap=false&includeList=' + includeList # first_url = default_first_url.format(pagination, usersSearchTerm, west, east, south, north, regionId, regionType, mapZoom, includeList) # print(first_case_url) # return default_page_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{"currentPage":' + pagination + '},' + '"usersSearchTerm":"' + usersSearchTerm + '","mapBounds":{"west":' + west + ',"east":' + east + ',"south":' + south + ',"north":' + north + '},"regionSelection":[{"regionId":' + regionId + ',"regionType":' + regionType + '}],"isMapVisible":true,"mapZoom":' + mapZoom + ',"filterState":{"sort":{"value":"globalrelevanceex"},"price":{"max":' + priceMax + ',"min":' + priceMin + '},"monthlyPayment":{"max":' + monthlyPayMax + ',"min":' + monthlyPayMin + '}},"isListVisible":true}&includeMap=false&includeList=' + includeList counts = 1 for page in range(1, 11): default_page_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{"currentPage":' + str( page ) + '},' + '"usersSearchTerm":"' + usersSearchTerm + '","mapBounds":{"west":' + west + ',"east":' + east + ',"south":' + south + ',"north":' + north + '},"regionSelection":[{"regionId":' + regionId + ',"regionType":' + regionType + '}],"isMapVisible":true,"mapZoom":' + mapZoom + ',"filterState":{"sort":{"value":"globalrelevanceex"},"price":{"max":' + priceMax + ',"min":' + priceMin + '},"monthlyPayment":{"max":' + monthlyPayMax + ',"min":' + monthlyPayMin + '}},"isListVisible":true}&includeMap=false&includeList=' + includeList if page == 1: url = first_case_url else: url = default_page_url response = requests.get(url, headers=header) result = response.json() properties_infos = result["searchResults"]["listResults"] print(len(properties_infos)) for i in range(0, len(properties_infos)): data_base = [] property_url = properties_infos[i]["detailUrl"] status_text = properties_infos[i]["statusText"] print(status_text, counts) counts += 1 try: street_add = properties_infos[i]["hdpData"]["homeInfo"][ "streetAddress"] except: street_add = "" try: city = properties_infos[i]["hdpData"]["homeInfo"]["city"] except: city = "" try: state = properties_infos[i]["hdpData"]["homeInfo"]["state"] except: state = "" try: zipcode = properties_infos[i]["hdpData"]["homeInfo"]["zipcode"] except: zipcode = "" property_address = street_add + ", " + city + ", " + state + " " + zipcode try: bathrooms = properties_infos[i]["hdpData"]["homeInfo"][ "bathrooms"] except: bathrooms = "" try: bedrooms = properties_infos[i]["hdpData"]["homeInfo"][ "bedrooms"] except: bedrooms = "" try: tax_assessed_value = properties_infos[i]["hdpData"][ "homeInfo"]["taxAssessedValue"] except: tax_assessed_value = "" try: zestimate = properties_infos[i]["hdpData"]["homeInfo"][ "zestimate"] except: zestimate = "" try: rent_zestimate = properties_infos[i]["hdpData"]["homeInfo"][ "rentZestimate"] except: rent_zestimate = "" try: home_type = properties_infos[i]["hdpData"]["homeInfo"][ "homeType"] except: home_type = "" if "by owner" in status_text: print("--------------------------------------------------> : ", i + 1) driver.get(property_url) time.sleep(10) # street_add = driver.find_element_by_xpath("//h1[@class='ds-address-container']/span[1]").text # property_address = street_add + ", " + city + ", " + state + " " + zipcode # phone_number = driver.find_element_by_xpath("//span[@class='listing-field']").text phones = re.findall(r'[(][\d]{3}[)][ ]?[\d]{3}-[\d]{4}', driver.page_source) for phone in range(1, len(phones) + 1): phone_number = phones[phone - 1] features_labels = driver.find_elements_by_xpath( "//ul[@class='ds-home-fact-list']//span[contains(@class, 'ds-standard-label') and contains(@class, 'ds-home-fact-label')]" ) features_infos = driver.find_elements_by_xpath( "//ul[@class='ds-home-fact-list']//span[contains(@class, 'ds-body') and contains(@class, 'ds-home-fact-value')]" ) parking = "" year_built = "" hoa = "" heating = "" lot = "" cooling = "" price_sqft = "" for feature_label, feature_info in zip(features_labels, features_infos): feature_label_txt = feature_label.text if 'Parking' in feature_label_txt: parking = feature_info.text elif 'Year built' in feature_label_txt: year_built = feature_info.text elif 'HOA' in feature_label_txt: hoa = feature_info.text elif 'Heating' in feature_label_txt: heating = feature_info.text elif 'Lot' in feature_label_txt: lot = feature_info.text elif 'Cooling' in feature_label_txt: cooling = feature_info.text elif 'Price/' in feature_label_txt: price_sqft = feature_info.text print("Property Address--------------------> : ", property_address) print("Property Url------------------------> : ", property_url) print("Property Status---------------------> : ", status_text) print("Owner Phone Number------------------> : ", phone_number) print("BathRooms---------------------------> : ", bathrooms) print("BedRooms----------------------------> : ", bedrooms) print("Tax Assessed Value------------------> : ", tax_assessed_value) print("Zestimate---------------------------> : ", zestimate) print("Rent Zestimate----------------------> : ", rent_zestimate) print("Home Type---------------------------> : ", home_type) print("Parking-----------------------------> : ", parking) print("Year Built--------------------------> : ", year_built) print("HOA---------------------------------> : ", hoa) print("Heating-----------------------------> : ", heating) print("Lot---------------------------------> : ", lot) print("Cooling-----------------------------> : ", cooling) print("Price Sqft--------------------------> : ", price_sqft) string_id = property_address + status_text + phone_number m = hashlib.md5() m.update(string_id.encode('utf8')) identifier = m.hexdigest() print("hash-------------------->", identifier) create_time = str(datetime.datetime.now()) update_time = "" insertdb = InsertDB() data_base.append( (property_address, street_add, city, state, zipcode, status_text, phone_number, bathrooms, bedrooms, tax_assessed_value, zestimate, rent_zestimate, home_type, parking, year_built, hoa, heating, lot, cooling, price_sqft, identifier, create_time, update_time)) insertdb.insert_document(data_base, table_name)
def main(htmlstring, driver): currentDT = datetime.datetime.now() table_name = "maricopa" header = { 'accept': '*/*', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'en-US,en;q=0.9,ko;q=0.8', 'referer': 'https://www.zillow.com/homes/85139_rb/', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36' } url = "https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState=%7B%22pagination%22%3A%7B%7D%2C%22mapBounds%22%3A%7B%22west%22%3A-113.68680328906252%2C%22east%22%3A-110.68753571093752%2C%22south%22%3A31.344189534984903%2C%22north%22%3A35.173746183166216%7D%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A2402%2C%22regionType%22%3A4%7D%5D%2C%22isMapVisible%22%3Atrue%2C%22mapZoom%22%3A8%2C%22filterState%22%3A%7B%22sort%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%7D%2C%22isListVisible%22%3Atrue%7D" # url = "https://www.zillow.com/homes/85138_rb/" response = requests.get(url, headers=header) result = response.json() properties_infos = result["searchResults"]["mapResults"] print(len(properties_infos)) for i in range(0, len(properties_infos)): data_base = [] property_url = "https://www.zillow.com/" + properties_infos[i][ "detailUrl"] status_text = properties_infos[i]["statusText"] try: street_add = properties_infos[i]["hdpData"]["homeInfo"][ "streetAddress"] except: street_add = "" try: city = properties_infos[i]["hdpData"]["homeInfo"]["city"] except: city = "" try: state = properties_infos[i]["hdpData"]["homeInfo"]["state"] except: state = "" try: zipcode = properties_infos[i]["hdpData"]["homeInfo"]["zipcode"] except: zipcode = "" property_address = street_add + ", " + city + ", " + state + " " + zipcode if "by owner" in status_text: print("--------------------------------------------------> : ", i + 1) print("Property Address--------------------> : ", property_address) print("Property Url------------------------> : ", property_url) print("Property Status---------------------> : ", status_text) driver.get(property_url) time.sleep(10) # phone_number = driver.find_element_by_xpath("//span[@class='listing-field']").text phones = re.findall(r'[(][\d]{3}[)][ ]?[\d]{3}-[\d]{4}', driver.page_source) for phone in range(1, len(phones) + 1): phone_number = phones[phone - 1] print("Owner Phone Number------------------> : ", phone_number) with open("Zillow_Maricopan_AZ_only500.csv", "a", newline="", encoding="utf-8") as f: writer = csv.writer(f) writer.writerow([property_address, status_text, phone_number]) string_id = property_address + status_text + phone_number m = hashlib.md5() m.update(string_id.encode('utf8')) identifier = m.hexdigest() print("hash-------------------->", identifier) create_time = str(datetime.datetime.now()) update_time = "" insertdb = InsertDB() data_base.append((property_address, street_add, city, state, zipcode, status_text, phone_number, identifier, create_time, update_time)) insertdb.insert_document(data_base, table_name)
def main(htmlstring, driver): table_name = "maricopa_30_08_2020" header = { 'accept': '*/*', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'en-US,en;q=0.9,ko;q=0.8', 'cookie': 'zguid=23|%2410ab80e6-80db-4e0a-9f70-2449ca972d74; _ga=GA1.2.759159145.1599348167; zjs_user_id=null; zjs_anonymous_id=%2210ab80e6-80db-4e0a-9f70-2449ca972d74%22; _gcl_au=1.1.607943717.1599348169; _pxvid=be9ff2f0-efce-11ea-9652-0242ac12000b; __gads=ID=cab593cad6cbce43:T=1599348200:S=ALNI_MaFYrYCZZvPIITKUEoEDXGvXSRYwQ; _gid=GA1.2.1287304564.1599556314; _pin_unauth=dWlkPU9EUXdZamxrTldJdE9ESTBNUzAwWXprMExXSXdNekl0TkdWak0yWTFNVEE1TldJeSZycD1abUZzYzJV; ki_r=; ki_s=; _fbp=fb.1.1599562363584.1440832488; g_state={"i_p":1599570378147,"i_l":1}; ki_t=1599556892885%3B1599556892885%3B1599563330503%3B1%3B19; JSESSIONID=62F47C1DAFBF00B3DB7B301BEA3E6586; zgsession=1|8840c1ee-f8a6-43d7-9a7b-3169df33c987; _pxff_cc=U2FtZVNpdGU9TGF4Ow==; _pxff_rf=1; _pxff_fp=1; _pxff_bsco=1; _px3=6d722620cec81d0df86c8eff4b631bdd93cef163fb0a14808e80f81013747454:M7trNae6CpAztMArZT97P3Vy9jFLz9FuEZ5p2efYpXeqOJC7Bw+xzsVGxArAYe+PM+vQKNuEI3qytjutx2UEXg==:1000:M1Vo/kdU1lI8Zqky6jJnuwSu45xHxX8ueCLKUiW6KX8rNR+VWAORLQi+1ns4dhilOU7gSCJfJmToj1SeyKN49kHZQZIQ0wSFeFtn+txzkIo/fhFAr2Cq7WvjCVWw7GBx8F3JIjMqHf1BZAAFg0YXqy/IVuCFhvIioSyK35nkm4A=; _gat=1; KruxPixel=true; DoubleClickSession=true; _uetsid=f44fc66ca5c392a6859170ed776b6ae9; _uetvid=dc708dafb2b6d91ab6c6923ac1ae6673; AWSALB=3gLhoP6QCdmf4zskymQ7ej/kbqzRHNkv+QNQMFmS6Y7S9pENaOusdnQVhFHWm1W9z8/1Og/WmO8JK63ys0wmi6ZNwRc4SN8lf4pcoyrm+nj8lLAPLRDIqMaYAEte; AWSALBCORS=3gLhoP6QCdmf4zskymQ7ej/kbqzRHNkv+QNQMFmS6Y7S9pENaOusdnQVhFHWm1W9z8/1Og/WmO8JK63ys0wmi6ZNwRc4SN8lf4pcoyrm+nj8lLAPLRDIqMaYAEte; search=6|1602203173818%7Crb%3DMaricopa%252C-AZ%26rect%3D33.203401%252C-111.882231%252C32.788612%252C-112.512953%26disp%3Dmap%26mdm%3Dauto%26sort%3Ddays%26pt%3Dpmf%252Cpf%26fs%3D1%26fr%3D0%26rs%3D0%26ah%3D0%26singlestory%3D0%26abo%3D0%26garage%3D0%26pool%3D0%26ac%3D0%26waterfront%3D0%26finished%3D0%26unfinished%3D0%26cityview%3D0%26mountainview%3D0%26parkview%3D0%26waterview%3D0%26hoadata%3D1%26zillow-owned%3D0%263dhome%3D0%09%0932697%09%09%09%09%09%09', 'sec-fetch-dest': 'empty', 'sec-fetch-mode': 'cors', 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36' } pagination = "" usersSearchTerm = "85006" west = "-112.07973577801513" east = "-112.01665022198486" south = "33.43522122804253" north = "33.494937169247144" regionId = "94722" regionType = "7" mapZoom = "14" includeList = "true" # https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={%22pagination%22:{},%22usersSearchTerm%22:%2285006%22,%22mapBounds%22:{%22west%22:-112.07973577801513,%22east%22:-112.01665022198486,%22south%22:33.43522122804253,%22north%22:33.494937169247144},%22regionSelection%22:[{%22regionId%22:94722,%22regionType%22:7}],%22isMapVisible%22:true,%22mapZoom%22:14,%22filterState%22:{%22sort%22:{%22value%22:%22globalrelevanceex%22}},%22isListVisible%22:true}&includeMap=false&includeList=true default_first_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{0},"usersSearchTerm":"{1}","mapBounds":{"west":{2},"east":{3},"south":{4},"north":{5}},"regionSelection":[{"regionId":{6},"regionType":{7}}],"isMapVisible":true,"mapZoom":{8},"filterState":{"sort":{"value":"globalrelevanceex"}},"isListVisible":true}&includeMap=false&includeList={9}' first_case_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{' + pagination + '},' + '"usersSearchTerm":"' + usersSearchTerm + '","mapBounds":{"west":' + west + ',"east":' + east + ',"south":' + south + ',"north":' + north + '},"regionSelection":[{"regionId":' + regionId + ',"regionType":' + regionType + '}],"isMapVisible":true,"mapZoom":' + mapZoom + ',"filterState":{"sort":{"value":"globalrelevanceex"}},"isListVisible":true}&includeMap=false&includeList=' + includeList # first_url = default_first_url.format(pagination, usersSearchTerm, west, east, south, north, regionId, regionType, mapZoom, includeList) print(first_case_url) # return default_page_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{"currentPage":' + pagination + '},' + '"usersSearchTerm":"' + usersSearchTerm + '","mapBounds":{"west":' + west + ',"east":' + east + ',"south":' + south + ',"north":' + north + '},"regionSelection":[{"regionId":' + regionId + ',"regionType":' + regionType + '}],"isMapVisible":true,"mapZoom":' + mapZoom + ',"filterState":{"sort":{"value":"globalrelevanceex"}},"isListVisible":true}&includeMap=false&includeList=' + includeList counts = 1 for page in range(1, 4): default_page_url = 'https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState={"pagination":{"currentPage":' + str( page ) + '},' + '"usersSearchTerm":"' + usersSearchTerm + '","mapBounds":{"west":' + west + ',"east":' + east + ',"south":' + south + ',"north":' + north + '},"regionSelection":[{"regionId":' + regionId + ',"regionType":' + regionType + '}],"isMapVisible":true,"mapZoom":' + mapZoom + ',"filterState":{"sort":{"value":"globalrelevanceex"}},"isListVisible":true}&includeMap=false&includeList=' + includeList if page == 1: url = first_case_url else: url = default_page_url response = requests.get(url, headers=header) result = response.json() properties_infos = result["searchResults"]["listResults"] print(len(properties_infos)) for i in range(0, len(properties_infos)): data_base = [] property_url = properties_infos[i]["detailUrl"] status_text = properties_infos[i]["statusText"] print(status_text, counts) counts += 1 try: street_add = properties_infos[i]["hdpData"]["homeInfo"][ "streetAddress"] except: street_add = "" try: city = properties_infos[i]["hdpData"]["homeInfo"]["city"] except: city = "" try: state = properties_infos[i]["hdpData"]["homeInfo"]["state"] except: state = "" try: zipcode = properties_infos[i]["hdpData"]["homeInfo"]["zipcode"] except: zipcode = "" property_address = street_add + ", " + city + ", " + state + " " + zipcode if "by owner" in status_text: print("--------------------------------------------------> : ", i + 1) driver.get(property_url) time.sleep(10) # street_add = driver.find_element_by_xpath("//h1[@class='ds-address-container']/span[1]").text # property_address = street_add + ", " + city + ", " + state + " " + zipcode # phone_number = driver.find_element_by_xpath("//span[@class='listing-field']").text phones = re.findall(r'[(][\d]{3}[)][ ]?[\d]{3}-[\d]{4}', driver.page_source) for phone in range(1, len(phones) + 1): phone_number = phones[phone - 1] print("Property Address--------------------> : ", property_address) print("Property Url------------------------> : ", property_url) print("Property Status---------------------> : ", status_text) print("Owner Phone Number------------------> : ", phone_number) string_id = property_address + status_text + phone_number m = hashlib.md5() m.update(string_id.encode('utf8')) identifier = m.hexdigest() print("hash-------------------->", identifier) create_time = str(datetime.datetime.now()) update_time = "" insertdb = InsertDB() data_base.append((property_address, street_add, city, state, zipcode, status_text, phone_number, identifier, create_time, update_time)) insertdb.insert_document(data_base, table_name)