def CommsController(self,url): self.craw_controller(url) while self.comms.has_new_url(): comm = self.comms.get_new_url comm = ToolsBox.clear_comm(comm) c1,c2 = self.comms.get_quantity() comm_url = 'https://xm.ke.com/ershoufang/pg1rs' + parse.quote(comm) + '/' print('*******{0}/{1}:{2}*********'.format(self.comm_count,c1+c2,comm)) url_list = [] url_list.append(comm_url) self.craw_controller(url_list) self.comm_count += 1 self.total = self.total + self.outputer.out_mysql() print('==================共抓取{0}个记录=================='.format(self.total))
def CommsController(self, url): self.craw_controller(url) while self.comms.has_new_url(): comm = self.comms.get_new_url comm = ToolsBox.clear_comm(comm) c1, c2 = self.comms.get_quantity() comm_url = 'https://xm.anjuke.com/sale/p1-rd1/?kw=' + parse.quote( comm) + '&from_url=kw_final#filtersort' # comm_url = 'https://xm.anjuke.com/sale/p1-rd1/?kw=' + parse.quote(comm) + '&from_url=kw_final#filtersort' print('*******{0}/{1}:{2}*********'.format(self.comm_count, c1 + c2, comm)) url_list = [] url_list.append(comm_url) self.craw_controller(url_list) self.comm_count += 1 self.total = self.total + self.outputer.out_mysql() print('==================共抓取{0}个记录=================='.format( self.total))
def CommsController(self, url): self.craw_controller(url) while self.comms.has_new_url(): comm = self.comms.get_new_url comm = ToolsBox.clear_comm(comm) c1, c2 = self.comms.get_quantity() # comm_url = "http://xm.58.com/ershoufang/?key=" + (comm) comm_url = "https://xm.58.com/ershoufang/?key=" + quote( comm, safe='/:?=') print('*******{0}/{1}:{2}*********'.format(self.comm_count, c1 + c2, comm)) url_list = [] url_list.append(comm_url) self.craw_controller(url_list) self.comm_count += 1 self.total = self.total + self.outputer.out_mysql() print('==================共抓取{0}个记录=================='.format( self.total))