def run(self): metaCrawler = VenueMetaCrawler() _counter = 0 while True: id = self.queue.get() _v = metaCrawler.grab_meta_data(id) print 'checking ',_v['venue']['name'] self.queue.task_done() """
def submit(self, downloading_locker): _meta_crawler = VenueMetaCrawler() _foursqaure_photo_crawler = VenuePhotoCrawlerFoursquare() for id in self.venue_ids: self.venue_count += 1 self.status = 'Fetching meta data for venue id = ' + str(id) _meta_crawler.grab_meta_data(id) #sleep(1) #self.status = 'Fetching foursquare photo data for venue id = ' +str(id) #_foursqaure_photo_crawler.grab_photo(id) #sleep(1) self.status = 'Finished' downloading_locker.clear() return "job done"
def submit(self, downloading_locker): _meta_crawler = VenueMetaCrawler() _foursqaure_photo_crawler = VenuePhotoCrawlerFoursquare() for id in self.venue_ids: self.venue_count += 1 self.status = 'Fetching meta data for venue id = '+str(id) _meta_crawler.grab_meta_data(id) #sleep(1) #self.status = 'Fetching foursquare photo data for venue id = ' +str(id) #_foursqaure_photo_crawler.grab_photo(id) #sleep(1) self.status = 'Finished' downloading_locker.clear() return "job done"