def handle(self, *args, **options): process_recursively = options.get('process_recursively') if (len(args) == 0): for this_callsign in mandated_stations: # There's confusion as to when files need to get uploaded for year in (2014): print "\n\nProcessing %s : %s - logs to: %s" % (year, this_callsign, FCC_SCRAPER_LOG_DIRECTORY) url = "https://stations.fcc.gov/station-profile/%s/political-files/browse->%s" % (this_callsign, year) this_folder = folder_placeholder(url, 'root', this_callsign) this_folder.process(process_recursively) elif len(args) > 0: url_array = [] for arg in args: url_to_process = args[0] url_to_process = url_to_process.replace("%3E", ">") (callSign, pathArray) = parse_folder_url(url_to_process) if not callSign: raise CommandError("Couldn't find callsign in folder URL: %s" % (url_to_process)) url_array.append({'url':url_to_process, 'callSign':callSign}) for this_url in url_array: print "\n\nProcessing %s - logs to: %s" % (this_url, FCC_SCRAPER_LOG_DIRECTORY) this_folder = folder_placeholder(this_url['url'], 'manual process', this_url['callSign']) this_folder.process(process_recursively) else: raise CommandError('Invalid arguments')
def handle(self, *args, **options): process_recursively = options.get('process_recursively') if (len(args) == 0): all_station_values = PDF_File.objects.all().order_by('callsign').values('callsign').distinct() for this_station in all_station_values: this_callsign = this_station['callsign'] for year in YEARS_WE_CARE_ABOUT: print "\n\nProcessing %s : %s - logs to: %s" % (year, this_callsign, FCC_SCRAPER_LOG_DIRECTORY) url = "https://stations.fcc.gov/station-profile/%s/political-files/browse->%s" % (this_callsign, year) this_folder = folder_placeholder(url, 'root', this_callsign) this_folder.process_feeds(process_recursively) elif len(args) > 0: url_array = [] for this_callsign in args: for year in YEARS_WE_CARE_ABOUT: print "\n\nProcessing %s : %s - logs to: %s" % (year, this_callsign, FCC_SCRAPER_LOG_DIRECTORY) url = "https://stations.fcc.gov/station-profile/%s/political-files/browse->%s" % (this_callsign, year) this_folder = folder_placeholder(url, 'root', this_callsign) this_folder.process_feeds(process_recursively) else: raise CommandError('Invalid arguments')
def handle(self, *args, **options): process_recursively = options.get('process_recursively') if (len(args) == 0): #all_station_values = PDF_File.objects.all().order_by('callsign').values('callsign').distinct() all_station_values = [ 'KYW-TV', 'WCAU', 'WFMZ', 'WMGM-TV', 'WPHL-TV', 'WPSG', 'WPVI-TV', 'WTVE', 'WTXF-TV', 'WUVP-DT', 'WWSI' ] for this_station in all_station_values: #this_callsign = this_station['callsign'] this_callsign = this_station for year in YEARS_WE_CARE_ABOUT: print "\n\nProcessing %s : %s - logs to: %s" % ( year, this_callsign, FCC_SCRAPER_LOG_DIRECTORY) url = "https://stations.fcc.gov/station-profile/%s/political-files/browse->%s" % ( this_callsign, year) this_folder = folder_placeholder(url, 'root', this_callsign) this_folder.process_feeds(process_recursively) elif len(args) > 0: url_array = [] for this_callsign in args: for year in YEARS_WE_CARE_ABOUT: print "\n\nProcessing %s : %s - logs to: %s" % ( year, this_callsign, FCC_SCRAPER_LOG_DIRECTORY) url = "https://stations.fcc.gov/station-profile/%s/political-files/browse->%s" % ( this_callsign, year) this_folder = folder_placeholder(url, 'root', this_callsign) this_folder.process_feeds(process_recursively) else: raise CommandError('Invalid arguments')
def handle(self, *args, **options): process_recursively = options.get('process_recursively') if (len(args) == 0): for this_callsign in mandated_stations: # There's confusion as to when files need to get uploaded for year in (2014): print "\n\nProcessing %s : %s - logs to: %s" % ( year, this_callsign, FCC_SCRAPER_LOG_DIRECTORY) url = "https://stations.fcc.gov/station-profile/%s/political-files/browse->%s" % ( this_callsign, year) this_folder = folder_placeholder(url, 'root', this_callsign) this_folder.process(process_recursively) elif len(args) > 0: url_array = [] for arg in args: url_to_process = args[0] url_to_process = url_to_process.replace("%3E", ">") (callSign, pathArray) = parse_folder_url(url_to_process) if not callSign: raise CommandError( "Couldn't find callsign in folder URL: %s" % (url_to_process)) url_array.append({'url': url_to_process, 'callSign': callSign}) for this_url in url_array: print "\n\nProcessing %s - logs to: %s" % ( this_url, FCC_SCRAPER_LOG_DIRECTORY) this_folder = folder_placeholder(this_url['url'], 'manual process', this_url['callSign']) this_folder.process(process_recursively) else: raise CommandError('Invalid arguments')