if __name__ == '__main__':
''' This script should be called in the following way:
$ python wsj_article_urls.py 'startdate' 'enddate'
'''
start_date, end_date = argv[1], argv[2]
start_datetime = datetime.datetime.strptime(start_date, '%Y-%m-%d')
end_datetime = datetime.datetime.strptime(end_date, '%Y-%m-%d')
while True:
print('Scraping WSJ URLs from {0} to {1}'.format(start_date, end_date))
# Get dates to search over
dates = get_dates(start_date, end_date)
urls = set()
for date in dates:
print(date)
urls = get_urls(date, urls)
# Convert urls set to a list and write to a txt file
file_path = '../url_files/{0}'.format(
get_file_name('wsj', start_date, end_date))
with open(file_path, 'w') as f:
f.write(json.dumps(list(urls)))
f.close()
start_datetime = start_datetime - datetime.timedelta(days=7)
num_bad_extractions += thread.result
return num_bad_extractions
if __name__=='__main__':
''' This script should be called in the following way:
$ python npr_scraper.py 'startdate' 'enddate' 'table (optional)'
'''
# Create MongoClient
client = MongoClient()
# Initialize the Database
db = client['election_analysis']
# Initialize table
# If a table name has been provided use that, otherwise initialize 'articles' table
if len(argv) > 3:
tab = db[argv[3]]
else:
tab = db['articles']
start_date, end_date = argv[1], argv[2]
print 'Scraping NPR from {0} to {1}'.format(start_date, end_date)
dates = get_dates(start_date, end_date)
keywords = get_keywords_2016()
num_bad_extractions = concurrent_scrape_npr(tab, keywords, dates)
print 'NPR Scraping Done...'
print 'Number of Bad Extractions = {0}'.format(num_bad_extractions)