def parse(self, response): count = 1 self.driver.get(response.url) self.driver.get_screenshot_as_file('sc.png') ul = self.driver.find_elements_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/div[1]/div/ul/li') for each in ul: count = count + 1 #self.driver.get_screenshot_as_file('sc1.png') #self.driver.find_element_by_xpath('//div[@class="mobile-only more"]/a').click() #self.driver.get_screenshot_as_file('sc1.png') #print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + str(count) we = next_weekday(datetime.datetime.today(), 2) WE = "%02d/%02d/%02d" % (we.month, we.day, we.year) for j in range(2, count + 1): for i in range(1, 41): try: date = self.driver.find_element_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/table/tbody/tr[%s]/td[3]' % i).text print date + ': ' + WE if date != WE: continue else: item = Item() item['sheriff_no'] = self.driver.find_element_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/table/tbody/tr[%s]/td[1]' % i).text item['sale_date'] = self.driver.find_element_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/table/tbody/tr[%s]/td[3]' % i).text item['case_no'] = self.driver.find_element_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/table/tbody/tr[%s]/td[2]' % i).text item['address'] = self.driver.find_element_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/table/tbody/tr[%s]/td[7]/a' % i).text item['upset'] = self.driver.find_element_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/table/tbody/tr[%s]/td[10]' % i).text item['att'] = self.driver.find_element_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/table/tbody/tr[%s]/td[5]' % i).text item['dfd'] = self.driver.find_element_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/table/tbody/tr[%s]/td[6]' % i).text item['plf'] = self.driver.find_element_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/table/tbody/tr[%s]/td[4]' % i).text #item['zipcode'] = self.driver.find_element_by_xpath('//table[1]/tbody/tr[%s]/td[9]' % i).text yield item except: break self.driver.find_element_by_xpath( '//*[@id="SheriffForclosuresWP"]/div/div[1]/div/ul/li[%s]/a' % j).click() self.driver.close()
def parse(self, response): self.driver.get(response.url) el = self.driver.find_element_by_xpath('//a[@href="/Sales/SalesSearch?countyId=15"]') el.click() we = next_weekday(datetime.today(), 2) WE = "%s/%s/%s" % (we.month, we.day, we.year) ''' now = datetime.now() n = "%s/%s/%s" % (now.month, now.day, now.year) N = time.strptime(n, "%m/%d/%Y") std = now + timedelta(days=90) STD = "%s/%s/%s" % (std.month, std.day, std.year) std = time.strptime(STD, "%m/%d/%Y") ''' for i in range(1, 1300): result = self.driver.find_element_by_xpath("//table/tbody/tr[%s]/td[1]/a" % i) date = self.driver.find_element_by_xpath("//table/tbody/tr[%s]/td[3]" % i).text #date_str = time.strptime(date, "%m/%d/%Y") if date != WE: continue else: result.click() item = Item() item['sheriff_no'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[1]/td[2]').text item['sale_date'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[3]/td[2]').text item['case_no'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[2]/td[2]').text item['address'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[6]/td[2]').text item['upset'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[8]/td[2]').text item['att'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[9]/td[2]').text item['dfd'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[5]/td[2]').text item['plf'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[4]/td[2]').text item['schd_data'] = self.driver.find_element_by_xpath( '//table[2]/tbody/tr[1]/td[2]').text yield item self.driver.back() self.driver.close()
def parse(self, response): self.driver.get(response.url) el = self.driver.find_element_by_xpath( '//a[@href="/Sales/SalesSearch?countyId=2"]') el.click() tu = next_weekday(datetime.datetime.today(), 1) TU = "%s/%s/%s" % (tu.month, tu.day, tu.year) for i in range(1, 1300): result = self.driver.find_element_by_xpath( "//table/tbody/tr[%s]/td[1]/a" % i) date = self.driver.find_element_by_xpath( "//table/tbody/tr[%s]/td[3]" % i).text if date != TU: continue else: result.click() item = Item() item['sheriff_no'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[1]/td[2]').text item['sale_date'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[3]/td[2]').text item['case_no'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[2]/td[2]').text item['address'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[6]/td[2]').text item['upset'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[8]/td[2]').text item['att'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[9]/td[2]').text item['att_ph'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[10]/td[2]').text item['dfd'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[5]/td[2]').text item['plf'] = self.driver.find_element_by_xpath( '//table[1]/tbody/tr[4]/td[2]').text item['schd_data'] = self.driver.find_element_by_xpath( '//table[2]/tbody/tr[1]/td[2]').text yield item self.driver.back() self.driver.close()