def upload_covid_all_forecasts(path_to_processed_model_forecasts):
# meta info
project_name = 'COVID-19 Forecasts'
project_obj = None
project_timezeros = []
forecasts = os.listdir(path_to_processed_model_forecasts)
conn = util.authenticate()
# Get all existing timezeros in the project
for project in conn.projects:
if project.name == project_name:
project_obj = project
for timezero in project.timezeros:
project_timezeros.append(timezero.timezero_date)
break
# Get model name
separator = '-'
dir_name = separator.join(
forecasts[0].split(separator)[3:]).split('.csv')[0]
metadata = metadata_dict_for_file(path_to_processed_model_forecasts +
'metadata-' + dir_name + '.txt')
model_name = metadata['model_name']
model = [
model for model in project_obj.models if model.name == model_name
][0]
# Get names of existing forecasts to avoid re-upload
existing_forecasts = [forecast.source for forecast in model.forecasts]
for forecast in forecasts:
# Skip if forecast is already on zoltar
if forecast in existing_forecasts:
continue
# Skip metadata text file
if '.txt' in forecast:
continue
with open(path_to_processed_model_forecasts + forecast) as fp:
# Get timezero and create timezero on zoltar if not existed
time_zero_date = forecast.split(dir_name)[0][:-1]
if time_zero_date not in project_timezeros:
try:
project_obj.create_timezero(time_zero_date)
except Exception as ex:
print(ex)
# Validate covid19 file
errors_from_validation = validate_quantile_csv_file(
path_to_processed_model_forecasts + forecast)
# Upload forecast
if "no errors" == errors_from_validation:
quantile_json, error_from_transformation = json_io_dict_from_quantile_csv_file(
fp, COVID19_TARGET_NAMES, covid19_row_validator)
if len(error_from_transformation) > 0:
print(error_from_transformation)
else:
try:
util.upload_forecast(conn,
quantile_json,
forecast,
project_name,
model_name,
time_zero_date,
overwrite=False)
except Exception as ex:
print(ex)
else:
print(errors_from_validation)
fp.close()
import yaml
import logging
import pickle
import hashlib
import json
logging.basicConfig(level=logging.DEBUG)
cwd_p = Path(__file__).parent.resolve()
all_forecasts = glob.glob('./data-processed/**/*-*.csv')
# pprint.pprint(all_forecasts)
# meta info
project_name = 'COVID-19 Forecasts'
project_obj = None
project_timezeros = []
conn = util.authenticate()
url = 'https://github.com/midas-network/covid19-scenario-modeling-hub/tree/master/data-processed/'
# all_forecasts = glob.glob('./data')
project_obj = [
project for project in conn.projects if project.name == project_name
][0]
project_timezeros = [
timezero.timezero_date for timezero in project_obj.timezeros
]
models = [model for model in project_obj.models]
model_abbrs = [model.abbreviation for model in models]
zoltar_forecasts = []
repo_forecasts = []
