def handle(self, *args, **options):
all_station_values = PDF_File.objects.all().order_by('callsign').values('callsign').distinct()
for this_station in all_station_values:
this_callsign = this_station['callsign']
feed_url = get_working_station_feed_url(this_callsign)
print "handling feed %s" % feed_url
handle_feed_url(feed_url)
def process_feeds(self, recursive=True):
try:
self.read_page()
self.parse()
# If we hit an error just log it and keep rolling.
except:
tb = traceback.format_exc()
message = "*** Error trying to process URL:%s ***\n%s" % (self.url, tb)
my_logger.warn(message)
print message
return
feed_url = get_feed_url_from_folder_url(self.url)
print "\n\n**** NOW HANDLING FEED URL: %s" % (feed_url)
handle_feed_url(feed_url)
if recursive:
for child in self.childfolders:
if (child['size'] > 0):
childfolder = folder_placeholder(child['url'], child['folder_class'], self.callSign, child['size'])
childfolder.process_feeds(recursive=recursive)
sleep(SCRAPE_DELAY_TIME)
def handle(self, *args, **options):
#all_station_values = PDF_File.objects.all().order_by('callsign').values('callsign').distinct()
all_station_values = PDF_File.objects.filter(
callsign__gte='KMGH-TV').order_by('callsign').values(
'callsign').distinct()
for this_station in all_station_values:
this_callsign = this_station['callsign']
feed_url = get_working_station_feed_url(this_callsign)
print "handling feed %s" % feed_url
handle_feed_url(feed_url)