def query_craig(self): if self.verbose: print('# query_craig()' ) if self.veryverbose: print( self.info['filters'] ) cl_a = CraigslistForSale( site=self.info['site'], area=self.info['area'], category=self.info['category'], filters=self.info['filters'] ) limit = 0 for result in cl_a.get_results(sort_by='newest'): record_time = car_util.time_object( result['datetime'],'%Y-%m-%d %H:%M' ) clean_time = car_util.time_object( self.info['since-date'],'%Y-%m-%dT%H:%M:%SZ' ) if record_time > clean_time: xCar = car_info.car( {'url': result['url']} ).update_info() # print('x' + str(xCar.info) ) while xCar.error != '': print('ErroR ' + str(xCar.error) + result['url']) if xCar.error == 408: # Request Timeout print( ' time problem, wait 3 and try again ' ) xCar = car_info.car( {'url': result['url']} ).update_info() time.sleep(3) if car.error == 404: # Request not found print( ' not found!! ') xCar.info['sold-date'] = datetime.datetime.utcnow() xCar.save_me() xCar.post_solr() break time.sleep(1) limit += 1
def query_mod(self,q, f, exe=False): results = self.solr.query(q , rows = 0,filters=f) num = results.results_count results = self.solr.query(q , rows = num,filters=f) print( q ) print( f ) print( str( results.results_count) + " results" ) for car in results.documents: xCar = car_info.car(car) print(xCar.info['title'],end='') if(xCar.info.get('_make',0)) != 0 : print("\t" + str(xCar.info['_make']),end='') if(xCar.info.get('_model',0)) != 0 : print("\t" + str(xCar.info['_model']),end='') if(xCar.info.get('_year',0)) != 0 : print("\t" + str(xCar.info['_year']),end='') print() if exe: xCar.info['_make'] = [] xCar.info['_model'] = [] xCar.info['_year'] = [] xCar.post_solr() print( str( results.results_count) + " results" )
if False: for term in known_cities: print( term ) # find all entries with term # tag entry city: term if False: # columns = ['price', 'location','title','make'], # columns = ['price', 'location','title','make'], url = "http://localhost:8983/solr/craig" solr = pysolarized.solr.Solr( url ) make = 'Toyota' results = solr.query("title:" + make, rows = 0) num = results.results_count results = solr.query("title:" + make, sort = ['price desc'], rows = num) print( str( results.results_count) + " results" ) for car in results.documents: xCar = car_info.car(car) xCar.info['_make'] = make # xCar.show_info() xCar.post_solr() print(xCar.info['title']) print('done') if True: run = tagger().tag_makes() #run = tagger().tag_models() #run = tagger().tag_years()