def handle(self, *args, **options): all_station_values = PDF_File.objects.all().order_by('callsign').values('callsign').distinct() for this_station in all_station_values: this_callsign = this_station['callsign'] feed_url = get_working_station_feed_url(this_callsign) print "handling feed %s" % feed_url handle_feed_url(feed_url)
def process_feeds(self, recursive=True): try: self.read_page() self.parse() # If we hit an error just log it and keep rolling. except: tb = traceback.format_exc() message = "*** Error trying to process URL:%s ***\n%s" % (self.url, tb) my_logger.warn(message) print message return feed_url = get_feed_url_from_folder_url(self.url) print "\n\n**** NOW HANDLING FEED URL: %s" % (feed_url) handle_feed_url(feed_url) if recursive: for child in self.childfolders: if (child['size'] > 0): childfolder = folder_placeholder(child['url'], child['folder_class'], self.callSign, child['size']) childfolder.process_feeds(recursive=recursive) sleep(SCRAPE_DELAY_TIME)
def handle(self, *args, **options): #all_station_values = PDF_File.objects.all().order_by('callsign').values('callsign').distinct() all_station_values = PDF_File.objects.filter( callsign__gte='KMGH-TV').order_by('callsign').values( 'callsign').distinct() for this_station in all_station_values: this_callsign = this_station['callsign'] feed_url = get_working_station_feed_url(this_callsign) print "handling feed %s" % feed_url handle_feed_url(feed_url)