# Iterate through all the sources, and for anything that is an AUTOMATIC_DOWNLOAD
# get the file from the source url and store it at the desired path.
automatic_downloads = config.read_config(
cc_by=True,
cc_by_sa=True,
google_tos=True,
cc_by_nc=True,
filter_by_fetch_method='AUTOMATIC_DOWNLOAD',
filter_no_load_func=False,
filter_no_data=False,
filter_not_approved=args.whitelist)
todays_date = datetime.today().strftime('%Y-%m-%d')
for k in automatic_downloads:
params = automatic_downloads[k]
source_url = params['fetch']['source_url']
path_for_today = path_utils.path_to_data_for_date(params, todays_date)
print('Downloading data for: ', k)
print('Source url: ', source_url)
out_dir = path_for_today['dir']
out_file = path_for_today['file']
out_path = os.path.join(out_dir, out_file)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
output = wget.download(source_url, out_path)
print('\nFile written to: ', output)
print('Done with fetch_automatic_downloads.py')
filter_not_approved=args.allowlist)
spreadsheet_file = 'hospitalizations.xlsx'
most_recent_spreadsheet = path_utils.most_recent_subdir(
path_utils.path_to('spreadsheets_dir'), spreadsheet_file)
if args.date:
spreadsheet_date = str(args.date[0])
else:
spreadsheet_date = str(most_recent_spreadsheet['date'])
spreadsheet_path = os.path.join(path_utils.path_to('spreadsheets_dir'),
spreadsheet_date, spreadsheet_file)
print('Fetching spreadsheet for date: ', spreadsheet_date)
print('Spreadsheet path: ', spreadsheet_path)
# This assumes that every data source with params['fetch']['method'] == 'SCRAPED' comes from a single spreadsheet.
# If that stops being the case, will need to update this.
for k in scraped:
params = scraped[k]
path_for_data = path_utils.path_to_data_for_date(params, spreadsheet_date)
df = pd.read_excel(spreadsheet_path, k)
print('Fetched data will be written to: ', path_for_data)
out_dir = path_for_data['dir']
out_file = path_for_data['file']
out_path = os.path.join(out_dir, out_file)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
df.to_csv(out_path, index=False)