def authenticate(env_user="******", env_pass="******"):
"""Authenticate the user ID and password for connection to Zoltar.
:param env_user: username of account in Zoltar
:param env_pass: password ""
"""
# Ensure environment variables exist
env_vars = [env_user, env_pass]
for var in env_vars:
if os.environ.get(var) == None:
print("\nERROR: Cannot locate environment variable: %s" % var)
print(
"\nPC users, try the command: set %s='<your zoltar username>'"
% var)
print(
"Mac users, try the command: export %s=<your zoltar username>"
% var)
print("Then, Refresh the command window\n")
return
# Authenticate Zoltar connection
try:
conn = ZoltarConnection()
conn.authenticate(os.environ.get(env_user), os.environ.get(env_pass))
return conn
except Exception as exc:
print(f"Error authenticating Zoltar credentials: {exc!r}.")
print(
f"Ensure the environment variables for your username and password are correct."
)
return print("ERROR")
def create_project_app():
"""Application that demonstrates project creation and deletion.
App args:
- zoltar_host: host to pass to ZoltarConnection(). This is typically "https://www.zoltardata.com" (no quotes and no
trailing '/') unless used for developer testing.
- project_config_file: configuration json file for the project of interest. see zoltar documentation for details,
esp. utils.project.create_project_from_json()
Required environment variables:
- 'Z_USERNAME': username of the account that has permission to access the resources in above app args
- 'Z_PASSWORD': password ""
"""
#host = sys.argv[1]
project_config_file = sys.argv[1]
conn = ZoltarConnection()
conn.authenticate(os.environ.get('Z_USERNAME'),
os.environ.get('Z_PASSWORD'))
with open(project_config_file) as fp:
project_dict = json.load(fp)
# delete existing project if found
project_dict = project_dict[0]
existing_project = [
project for project in conn.projects
if project.name == project_dict['name']
]
if existing_project:
existing_project = existing_project[0]
print(f"deleting existing project: {existing_project}")
existing_project.delete()
print("delete done")
# create new project
print(f"creating new project. project name={project_dict['name']}")
response = requests.post(
f'{conn.host}/api/projects/',
headers={'Authorization': f'JWT {conn.session.token}'},
json={'project_config': project_dict})
if response.status_code != 200: # HTTP_200_OK
raise RuntimeError(
f"status_code was not 200. status_code={response.status_code}, text={response.text}"
)
new_project_json = response.json()
new_project = Project(conn, new_project_json['url'])
print(f"created new project: {new_project}")
def upload_to_zoltar(site_key, is_local_host=False):
proj_config = read_config(CONFIG_LOCAL_PATH)
helpers = {
FTE: upload_to_zoltar_fte,
ECONOMIST: upload_to_zoltar_economist
}
if is_local_host:
conn = ZoltarConnection(LOCAL_HOST)
conn.authenticate(LOCAL_USERNAME, LOCAL_PASSWORD)
helpers[site_key](conn, proj_config)
else:
conn = util.authenticate()
helpers[site_key](conn, proj_config)
def test_is_token_expired(self):
# test an expired token
conn = mock_authenticate(ZoltarConnection(
'http://example.com')) # default token (mock_token) is expired
self.assertTrue(conn.session.is_token_expired())
# construct and test an unexpired token
token_split = MOCK_TOKEN.split(
'.') # 3 parts: header, payload, signature
old_header = token_split[0]
old_signature = token_split[1]
# round to exclude decimal portion - throws off some JWT tools:
ten_min_from_now = round((datetime.datetime.utcnow() +
datetime.timedelta(minutes=10)).timestamp())
new_payload = {
'user_id': 3,
'username': '******',
'exp': ten_min_from_now,
'email': ''
}
new_payload_json = json.dumps(new_payload)
payload_b64 = base64.b64encode(new_payload_json.encode('utf_8'))
unexpired_token = f"{old_header}.{payload_b64.decode('utf-8')}.{old_signature}"
conn.session.token = unexpired_token
self.assertFalse(conn.session.is_token_expired())
def test_json_for_uri_calls_re_authenticate_if_necessary(self):
with patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary') as re_auth_mock, \
patch('requests.get') as get_mock:
get_mock.return_value.status_code = 200
conn = mock_authenticate(ZoltarConnection('http://example.com'))
conn.json_for_uri('/')
re_auth_mock.assert_called_once()
def test_verify_instance_api_hits(self, json_for_uri_mock):
json_for_uri_mock.return_value = PROJECTS_LIST_DICTS
conn = ZoltarConnection('https://example.com')
projects = conn.projects
json_for_uri_mock.assert_called_once_with(
'https://example.com/api/projects/')
json_for_uri_mock.reset_mock()
models = projects[0].models
json_for_uri_mock.assert_called_once_with(
'http://example.com/api/project/3/models/')
json_for_uri_mock.reset_mock()
projects[0].units
json_for_uri_mock.assert_called_once_with(
'http://example.com/api/project/3/units/')
json_for_uri_mock.reset_mock()
projects[0].targets
json_for_uri_mock.assert_called_once_with(
'http://example.com/api/project/3/targets/')
json_for_uri_mock.reset_mock()
projects[0].timezeros
json_for_uri_mock.assert_called_once_with(
'http://example.com/api/project/3/timezeros/')
json_for_uri_mock.return_value = MODELS_LIST_DICTS[0]
json_for_uri_mock.reset_mock()
models[0].name
json_for_uri_mock.assert_not_called()
def test_create_timezero(self):
conn = mock_authenticate(ZoltarConnection('http://example.com'))
with patch('zoltpy.connection.ZoltarConnection.json_for_uri', return_value=PROJECTS_LIST_DICTS), \
patch('requests.post') as post_mock, \
patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary') as re_auth_mock:
project = conn.projects[0]
post_mock.return_value.status_code = 200
post_return_value = {
"id": 705,
"url": "http://example.com/api/timezero/705/",
"timezero_date": "2011-10-02",
"data_version_date": "2011-10-03",
"is_season_start": True,
"season_name": "2011-2012"
}
post_mock.return_value.json = MagicMock(
return_value=post_return_value)
project.create_timezero("2011-10-02", "2011-10-03", True,
"2011-2012")
post_mock.assert_called_once()
exp_timezero_config = dict(post_return_value) # copy
del exp_timezero_config['id']
del exp_timezero_config['url']
act_timezero_config = post_mock.call_args[1]['json'][
'timezero_config']
self.assertEqual(exp_timezero_config, act_timezero_config)
re_auth_mock.assert_called_once()
def test_query_with_ids(self, json_for_uri_mock):
def json_for_uri_mock_side_effect(*args, **kwargs):
return {
'http://example.com/api/projects/': PROJECTS_LIST_DICTS,
'http://example.com/api/project/3/models/': MODELS_LIST_DICTS,
'http://example.com/api/project/3/units/': UNITS_LIST_DICTS,
'http://example.com/api/project/3/targets/':
TARGETS_LIST_DICTS,
'http://example.com/api/project/3/timezeros/':
TIMEZEROS_LIST_DICTS
}[args[0]]
json_for_uri_mock.side_effect = json_for_uri_mock_side_effect
conn = mock_authenticate(ZoltarConnection('http://example.com'))
project = conn.projects[0]
# case: blue sky
input_exp_output_queries = [
({}, {}),
({
'models': ['docs forecast model'],
'units': ['location1', 'location2'],
'targets': ['pct next week', 'cases next week'],
'timezeros': ['2011-10-02', '2011-10-16'],
'types': ['point', 'quantile']
}, {
'models': [5],
'units': [23, 24],
'targets': [15, 16],
'timezeros': [5, 7],
'types': ['point', 'quantile']
}),
(
{
'models': ['doc_model_abbrev']
}, # 'models' can be either model name or abbreviation
{
'models': [5]
})
]
for input_query, exp_output_query in input_exp_output_queries:
act_output_query = project.query_with_ids(input_query)
self.assertEqual(exp_output_query, act_output_query)
# case: name not found
for query in [{
'models': ['bad model name']
}, {
'units': ['bad unit name']
}, {
'targets': ['bad target name']
}, {
'timezeros': ['1999-10-02']
}]:
with self.assertRaises(RuntimeError) as context:
project.query_with_ids(query)
self.assertIn('one or more', str(context.exception))
def test_authenticate(self):
conn = ZoltarConnection('')
username = '******'
password = '******'
mock_authenticate(conn, username, password)
with patch('requests.post') as post_mock:
post_mock.return_value.status_code = 200
post_mock.return_value.json = MagicMock(
return_value={'token': MOCK_TOKEN})
conn.authenticate(username, password)
self.assertEqual(username, conn.username)
self.assertEqual(password, conn.password)
self.assertIsInstance(conn.session, ZoltarSession)
self.assertEqual(MOCK_TOKEN, conn.session.token)
post_mock.assert_called_once_with('/api-token-auth/', {
'username': '******',
'password': '******'
})
def test_instances(self, json_for_uri_mock):
json_for_uri_mock.return_value = PROJECTS_LIST_DICTS
conn = ZoltarConnection()
# test ZoltarConnection.projects
projects = conn.projects # hits /api/projects/
self.assertEqual(2, len(projects))
project_0 = projects[0]
self.assertIsInstance(project_0, Project)
self.assertEqual("Docs Example Project", project_0.name)
# test Project.models
json_for_uri_mock.return_value = MODELS_LIST_DICTS
models = project_0.models # hits api/project/3/models/
self.assertEqual(1, len(models))
model_0 = models[0]
self.assertIsInstance(model_0, Model)
self.assertEqual("docs forecast model", model_0.name)
# test Project.units
json_for_uri_mock.return_value = UNITS_LIST_DICTS
units = project_0.units # hits api/project/3/units/
self.assertEqual(3, len(units))
unit_0 = units[0]
self.assertIsInstance(unit_0, Unit)
self.assertEqual("location1", unit_0.name)
# test Project.targets
json_for_uri_mock.return_value = TARGETS_LIST_DICTS
targets = project_0.targets # hits api/project/3/targets/
self.assertEqual(2, len(targets))
target_0 = targets[0]
self.assertIsInstance(target_0, Target)
self.assertEqual("pct next week", target_0.name)
# test Project.timezeros
json_for_uri_mock.return_value = TIMEZEROS_LIST_DICTS
timezeros = project_0.timezeros # hits api/project/3/timezeros/
self.assertEqual(3, len(timezeros))
timezero_0 = timezeros[0]
self.assertIsInstance(timezero_0, TimeZero)
self.assertEqual("2011-10-02", timezero_0.timezero_date)
# test Model.forecasts
json_for_uri_mock.return_value = FORECASTS_LIST_DICTS
forecasts = model_0.forecasts # hits api/model/5/forecasts/
self.assertEqual(1, len(forecasts))
forecast_0 = forecasts[0]
self.assertIsInstance(forecast_0, Forecast)
self.assertEqual("docs-predictions.json", forecast_0.source)
def test_delete_calls_re_authenticate_if_necessary(self):
with patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary') as re_auth_mock, \
patch('zoltpy.connection.ZoltarConnection.json_for_uri') as json_for_uri_mock, \
patch('requests.delete') as delete_mock:
json_for_uri_mock.return_value = PROJECTS_LIST_DICTS
delete_mock.return_value.status_code = 200
conn = mock_authenticate(ZoltarConnection('http://example.com'))
projects = conn.projects
projects[0].delete()
re_auth_mock.assert_called_once()
def test_create_model(self, json_for_uri_mock):
with open('examples/example-model-config.json') as fp:
model_config = json.load(fp)
model_config['url'] = 'http://example.com/api/model/5/'
conn = mock_authenticate(ZoltarConnection('http://example.com'))
project = Project(conn, 'http://example.com/api/project/3/')
with patch('requests.post') as post_mock, \
patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary') as re_auth_mock:
post_mock.return_value.status_code = 200
post_mock.return_value.json = MagicMock(return_value=model_config)
new_model = project.create_model(model_config)
self.assertEqual(1, post_mock.call_count)
self.assertEqual('http://example.com/api/project/3/models/',
post_mock.call_args[0][0])
self.assertIsInstance(new_model, Model)
re_auth_mock.assert_called_once()
def test_upload_truth_data(self, json_for_uri_mock):
conn = mock_authenticate(ZoltarConnection('http://example.com'))
project = Project(conn, 'http://example.com/api/project/3/')
with open('tests/job-2.json') as ufj_fp, \
open('tests/docs-ground-truth.csv') as csv_fp, \
patch('requests.post') as post_mock, \
patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary') as re_auth_mock:
job_json = json.load(ufj_fp)
post_mock.return_value.status_code = 200
post_mock.return_value.json = MagicMock(return_value=job_json)
act_job_json = project.upload_truth_data(csv_fp)
re_auth_mock.assert_called_once()
self.assertEqual(1, post_mock.call_count)
self.assertEqual('http://example.com/api/project/3/truth/',
post_mock.call_args[0][0])
self.assertIsInstance(act_job_json, Job)
self.assertEqual(job_json['url'], act_job_json.uri)
def test_upload_forecast(self, json_for_uri_mock):
conn = mock_authenticate(ZoltarConnection('http://example.com'))
with open('tests/job-2.json') as ufj_fp, \
open("examples/example-model-config.json") as fp, \
patch('requests.post') as post_mock, \
patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary') as re_auth_mock:
job_json = json.load(ufj_fp)
model_config = json.load(fp)
model_config['url'] = 'http://example.com/api/model/5/'
post_mock.return_value.status_code = 200
post_mock.return_value.json = MagicMock(return_value=job_json)
forecast_model = Model(conn, model_config['url'], model_config)
act_job_json = forecast_model.upload_forecast({}, None, None)
re_auth_mock.assert_called_once()
self.assertEqual(1, post_mock.call_count)
self.assertEqual('http://example.com/api/model/5/forecasts/',
post_mock.call_args[0][0])
self.assertIsInstance(act_job_json, Job)
self.assertEqual(job_json['url'], act_job_json.uri)
def test_submit_truth_query(self, json_for_uri_mock):
json_for_uri_mock.return_value = PROJECTS_LIST_DICTS
conn = mock_authenticate(ZoltarConnection('http://example.com'))
project = conn.projects[0]
with open('tests/job-submit-query.json') as job_submit_json_fp, \
patch('requests.post') as post_mock, \
patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary'):
# test submit
query = {} # all forecasts
job_submit_json = json.load(job_submit_json_fp)
post_mock.return_value.status_code = 200
post_mock.return_value.json = MagicMock(
return_value=job_submit_json)
job = project.submit_query(QueryType.TRUTH, query)
self.assertEqual('http://example.com/api/project/3/truth_queries/',
post_mock.call_args[0][0])
self.assertEqual(post_mock.call_args[1]['json'], {'query': query})
self.assertIsInstance(job, Job)
def test_forecasts_set_issued_at(self):
from tests.test_util import FORECAST_DICT # avoid circular imports
conn = mock_authenticate(ZoltarConnection(
'http://example.com')) # default token (mock_token) is expired
with patch('requests.patch') as patch_mock, \
patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary') as re_auth_mock, \
patch('zoltpy.connection.ZoltarConnection.json_for_uri') as json_for_uri_mock:
patch_mock.return_value.status_code = 200
forecast = Forecast(conn, "http://example.com/api/forecast/3/",
FORECAST_DICT)
self.assertEqual(FORECAST_DICT['issued_at'], forecast.issued_at)
new_issued_at = 'new issued_at' # type isn't checked locally, just remotely
new_forecast_dict = dict(FORECAST_DICT) # non-deep copy OK
new_forecast_dict['issued_at'] = new_issued_at
forecast.issued_at = new_issued_at # call setter. does not refresh
json_for_uri_mock.return_value = new_forecast_dict
forecast.refresh()
self.assertEqual(new_issued_at, forecast.issued_at)
def test_delete_forecast(self):
def json_for_uri_mock_side_effect(*args, **kwargs): # returns a sequence of return args
return {'http://example.com/api/projects/': PROJECTS_LIST_DICTS,
'http://example.com/api/project/3/models/': [MODEL_DICT],
'https://example.com/api/model/150/forecasts/': [FORECAST_DICT]}[args[0]]
conn = mock_authenticate(ZoltarConnection('http://example.com'))
with patch('zoltpy.connection.ZoltarConnection.json_for_uri') as json_for_uri_mock, \
patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary'), \
patch('zoltpy.connection.Forecast.delete') as delete_forecast_mock:
json_for_uri_mock.side_effect = json_for_uri_mock_side_effect
# case: finds existing_forecast
delete_forecast(conn, PROJECTS_LIST_DICTS[0]['name'], MODEL_DICT['name'], '2020-04-12')
self.assertEqual(1, delete_forecast_mock.call_count)
# case: does not find existing_forecast
delete_forecast_mock.reset_mock()
delete_forecast(conn, PROJECTS_LIST_DICTS[0]['name'], MODEL_DICT['name'], '2020-04-22')
self.assertEqual(0, delete_forecast_mock.call_count)
def test_download_forecast(self):
def json_for_uri_mock_side_effect(*args, **kwargs): # returns a sequence of return args
return {'http://example.com/api/projects/': PROJECTS_LIST_DICTS,
'http://example.com/api/project/3/models/': [MODEL_DICT],
'https://example.com/api/model/150/forecasts/': [FORECAST_DICT]}[args[0]]
conn = mock_authenticate(ZoltarConnection('http://example.com'))
with patch('zoltpy.connection.ZoltarConnection.json_for_uri') as json_for_uri_mock, \
patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary'), \
patch('zoltpy.connection.Forecast.data') as forecast_data_mock:
json_for_uri_mock.side_effect = json_for_uri_mock_side_effect
project_name = "Docs Example Project"
model_abbr = "60-contact"
timezero_date = "2020-04-12"
# case: blue sky
download_forecast(conn, project_name, model_abbr, timezero_date)
forecast_data_mock.assert_called_once()
# case: bad project name
forecast_data_mock.reset_mock()
with self.assertRaises(RuntimeError) as context:
download_forecast(conn, project_name + 'bad', model_abbr, timezero_date)
self.assertIn('found no project named', str(context.exception))
# case: bad model name
forecast_data_mock.reset_mock()
with self.assertRaises(RuntimeError) as context:
download_forecast(conn, project_name, model_abbr + 'bad', timezero_date)
self.assertIn('found no model named', str(context.exception))
# case: bad timezero_date
forecast_data_mock.reset_mock()
with self.assertRaises(RuntimeError) as context:
download_forecast(conn, project_name, model_abbr, "2020-02-22")
self.assertIn('found no forecast with timezero date', str(context.exception))
def test_edit_model(self, json_for_uri_mock):
conn = ZoltarConnection()
mock_authenticate(conn, '', '')
json_for_uri_mock.return_value = PROJECTS_LIST_DICTS
projects = conn.projects # hits /api/projects/
project_0 = projects[0]
json_for_uri_mock.return_value = MODELS_LIST_DICTS
models = project_0.models # hits api/project/3/models/
model_0 = models[0]
with open('examples/example-model-config.json') as fp:
model_config = json.load(fp)
# case: blue sky
with patch('requests.put') as put_mock:
put_mock.return_value.status_code = 200
model_0.edit(model_config)
put_mock.assert_called_once_with(
'http://example.com/api/model/5/',
data={'model_config': model_config},
headers={'Authorization': f'JWT {MOCK_TOKEN}'})
def test_submit_and_download_forecast_query(self, json_for_uri_mock):
json_for_uri_mock.return_value = PROJECTS_LIST_DICTS
conn = mock_authenticate(ZoltarConnection('http://example.com'))
project = conn.projects[0]
with open('tests/job-submit-query.json') as job_submit_json_fp, \
patch('requests.post') as post_mock, \
patch('zoltpy.connection.ZoltarConnection.re_authenticate_if_necessary'):
# test submit
query = {} # all forecasts
job_submit_json = json.load(job_submit_json_fp)
post_mock.return_value.status_code = 200
post_mock.return_value.json = MagicMock(
return_value=job_submit_json)
job = project.submit_query(QueryType.FORECASTS, query)
self.assertEqual(
'http://example.com/api/project/3/forecast_queries/',
post_mock.call_args[0][0])
self.assertEqual(post_mock.call_args[1]['json'], {'query': query})
self.assertIsInstance(job, Job)
# test download
json_for_uri_mock.reset_mock(return_value=True)
json_for_uri_mock.return_value.content = b'unit,target,class,value,cat,prob,sample,quantile,family,param1,param2,param3\r\nlocation1,season severity,bin,,moderate,0.1,,,,,,\r\n'
rows = job.download_data()
json_for_uri_mock.assert_called_once_with(
'http://127.0.0.1:8000/api/job/44/data/', False, 'text/csv')
self.assertEqual(2, len(rows))
exp_rows = [[
'unit', 'target', 'class', 'value', 'cat', 'prob', 'sample',
'quantile', 'family', 'param1', 'param2', 'param3'
],
[
'location1', 'season severity', 'bin', '',
'moderate', '0.1', '', '', '', '', '', ''
]]
self.assertEqual(exp_rows, rows)
def test_instances(self, json_for_uri_mock):
json_for_uri_mock.return_value = PROJECTS_LIST_DICTS
conn = ZoltarConnection()
# test ZoltarConnection.projects
projects = conn.projects # hits /api/projects/
self.assertEqual(2, len(projects))
project_0 = projects[0]
self.assertIsInstance(project_0, Project)
self.assertEqual("Docs Example Project", project_0.name)
# test Project.models
json_for_uri_mock.return_value = MODELS_LIST_DICTS
models = project_0.models # hits api/project/3/models/
self.assertEqual(1, len(models))
model_0 = models[0]
self.assertIsInstance(model_0, Model)
self.assertEqual("docs forecast model", model_0.name)
# test Project.units
json_for_uri_mock.return_value = UNITS_LIST_DICTS
units = project_0.units # hits api/project/3/units/
self.assertEqual(3, len(units))
unit_0 = units[0]
self.assertIsInstance(unit_0, Unit)
self.assertEqual("loc1", unit_0.abbreviation)
self.assertEqual("location1", unit_0.name)
# test Project.targets
json_for_uri_mock.return_value = TARGETS_LIST_DICTS
targets = project_0.targets # hits api/project/3/targets/
self.assertEqual(2, len(targets))
target_0 = targets[0]
self.assertIsInstance(target_0, Target)
self.assertEqual("pct next week", target_0.name)
# test Project.timezeros
json_for_uri_mock.return_value = TIMEZEROS_LIST_DICTS
timezeros = project_0.timezeros # hits api/project/3/timezeros/
self.assertEqual(3, len(timezeros))
timezero_0 = timezeros[0]
self.assertIsInstance(timezero_0, TimeZero) # "2011-10-02"
self.assertIsInstance(timezero_0.timezero_date, datetime.date)
self.assertIsInstance(timezero_0.data_version_date, datetime.date)
self.assertEqual(datetime.date(2011, 10, 2), timezero_0.timezero_date)
self.assertEqual(datetime.date(2011, 10, 22),
timezero_0.data_version_date)
timezero_1 = timezeros[1] # None
self.assertIsInstance(timezero_1.timezero_date, datetime.date)
self.assertEqual(datetime.date(2011, 10, 9), timezero_1.timezero_date)
self.assertEqual(None, timezero_1.data_version_date)
# test Project.created_at
json_for_uri_mock.return_value = {
"id": 44,
"url": "https://www.zoltardata.com/api/project/44/truth/",
"project": "https://www.zoltardata.com/api/project/44/",
"source": "zoltar-truth.csv",
"created_at": "2021-06-16T21:06:37.893283+00:00",
"issued_at": "2021-06-16T21:06:37.851554+00:00"
}
created_at = project_0.truth_created_at
self.assertIsInstance(created_at, datetime.datetime)
self.assertEqual(
dateutil.parser.parse("2021-06-16T21:06:37.893283+00:00"),
created_at)
issued_at = project_0.truth_issued_at
self.assertIsInstance(issued_at, datetime.datetime)
self.assertEqual(
dateutil.parser.parse("2021-06-16T21:06:37.851554+00:00"),
issued_at)
# test Model.forecasts
json_for_uri_mock.return_value = FORECASTS_LIST_DICTS
forecasts = model_0.forecasts # hits api/model/5/forecasts/
self.assertEqual(2, len(forecasts))
# test Model.latest_forecast
json_for_uri_mock.return_value = FORECASTS_LIST_DICTS
latest_forecast = model_0.latest_forecast # hits api/model/5/forecasts/
self.assertIsInstance(latest_forecast, Forecast)
self.assertEqual(
'2020-08-17',
latest_forecast.timezero.timezero_date.strftime(
YYYY_MM_DD_DATE_FORMAT))
forecast_0 = forecasts[0]
self.assertIsInstance(forecast_0, Forecast)
self.assertEqual('2020-08-17-COVIDhub-ensemble.csv', forecast_0.source)
import pprint
logger = logging.getLogger(__name__)
#TODO: Make these as environment variables
STAGING = False
# meta info
project_name = 'COVID-19 Forecasts'
project_obj = None
project_timezeros = []
# Is staging is set to True, use the staging server
if STAGING:
conn = ZoltarConnection(host='https://rl-zoltar-staging.herokuapp.com')
else:
conn = ZoltarConnection()
conn.authenticate(os.environ.get("Z_USERNAME"), os.environ.get("Z_PASSWORD"))
url = 'https://github.com/reichlab/covid19-forecast-hub/tree/master/data-processed/'
# mapping of variables in the metadata to the parameters in Zoltar
metadata_field_to_zoltar = {
'team_name': 'team_name',
'model_name': 'name',
'model_abbr': 'abbreviation',
'model_contributors': 'contributors',
'website_url': 'home_url',
'license': 'license',
'team_model_designation': 'notes',
import pandas as pd
from zoltpy.quantile_io import json_io_dict_from_quantile_csv_file
from zoltpy import util
from zoltpy.cdc_io import YYYY_MM_DD_DATE_FORMAT
from zoltpy.connection import ZoltarConnection
import os
project_name = 'COVID-19 Forecasts'
conn = ZoltarConnection()
conn.authenticate(os.environ.get("Z_USERNAME"), os.environ.get("Z_PASSWORD"))
project_obj = [
project for project in conn.projects if project.name == project_name
][0]
models = [model for model in project_obj.models]
forecasts = []
for model in models:
forecasts.extend(list(model.forecasts))
forecast_sources = [forecast.source for forecast in forecasts]
deleted_forecasts = pd.read_csv("deleted_forecasts.csv")
for forecast in deleted_forecasts['file_name']:
if forecast in forecast_sources:
print(forecast)
def create_model_app():
"""Application that demonstrates model creation and deletion.
App args:
- zoltar_host: host to pass to ZoltarConnection()
- project_name: name of Project to work with
- model_name: name of a ForecastModel to create and delete. it will be given demonstration data
Required environment variables:
- 'Z_USERNAME': username of the account that has permission to access the resources in above app args
- 'Z_PASSWORD': password ""
"""
host = sys.argv[1]
project_name = sys.argv[2]
model_name = sys.argv[3]
conn = ZoltarConnection(host)
conn.authenticate(os.environ.get('Z_USERNAME'),
os.environ.get('Z_PASSWORD'))
# delete existing model if found
project = [
project for project in conn.projects if project.name == project_name
]
if not project:
print(f"could not find project with model_name={project_name!r}")
return
# show all models prior to delete/create
project = project[0]
print(f'\n* "before" models in {project}')
for model in project.models:
print(f'- {model}')
existing_model = [
model for model in project.models if model.name == model_name
]
if existing_model:
existing_model = existing_model[0]
print(f"deleting existing model: {existing_model}")
existing_model.delete()
print("delete done")
# create new model
model_config = {
'name': 'a model_name',
'abbreviation': 'an abbreviation',
'team_name': 'a team_name',
'description': 'a description',
'home_url': 'http://example.com/',
'aux_data_url': 'http://example.com/'
}
print(
f"creating new model. project={project}, model_config={model_config}")
new_model = project.create_model(model_config)
print(f"created new model: {new_model}")
# show all models after delete/create
project.refresh()
print(f'\n* "after" models in {project}')
for model in project.models:
print(f'- {model}')
def test_create_timezero(self):
conn = mock_authenticate(ZoltarConnection('http://example.com'))
with patch('zoltpy.connection.ZoltarConnection.json_for_uri',
return_value=PROJECTS_LIST_DICTS):
project = conn.projects[0]
# test valid arg combinations
valid_args_tuples = [("2011-10-02", None, False, ''),
("2011-10-02", "2011-10-03", False, ''),
("2011-10-02", "2011-10-03", True,
"'tis the season")]
for timezero_date, data_version_date, is_season_start, season_name in valid_args_tuples:
with patch('requests.post') as post_mock:
post_mock.return_value.status_code = 200
post_mock.return_value.json = MagicMock(
return_value={
"url": "http://example.com/api/timezero/497/"
})
try:
project.create_timezero(timezero_date, data_version_date,
is_season_start, season_name)
post_mock.assert_called_once()
except Exception as ex:
self.fail(f"unexpected exception: {ex}")
# test valid POST args
with patch('requests.post') as post_mock:
post_mock.return_value.status_code = 200
post_return_value = {
"id": 705,
"url": "http://example.com/api/timezero/705/",
"timezero_date": "2011-10-02",
"data_version_date": "2011-10-03",
"is_season_start": True,
"season_name": "2011-2012"
}
post_mock.return_value.json = MagicMock(
return_value=post_return_value)
try:
project.create_timezero("2011-10-02", "2011-10-03", True,
"2011-2012")
post_mock.assert_called_once()
exp_timezero_config = dict(post_return_value) # copy
del exp_timezero_config['id']
del exp_timezero_config['url']
act_timezero_config = post_mock.call_args[1]['json'][
'timezero_config']
self.assertEqual(exp_timezero_config, act_timezero_config)
except Exception as ex:
self.fail(f"unexpected exception: {ex}")
# test yes is_season_start but no season_name
with self.assertRaises(RuntimeError) as context:
project.create_timezero("2011-10-02", "2011-10-03", True)
self.assertIn(
'season_name not found but is required when is_season_start is passed',
str(context.exception))
# test no is_season_start but yes season_name
with self.assertRaises(RuntimeError) as context:
project.create_timezero("2011-10-02", False, "'tis the season")
self.assertIn(
'season_name not found but is required when is_season_start is passed',
str(context.exception))
def zoltar_connection_app():
"""
Application demonstrating use of the library at the ZoltarConnection level (rather than using the package's
higher-level functions such as delete_forecast(), etc.)
- App args: None
- Required environment variables:
- 'Z_HOST': Zoltar host to connect to. typically "https://www.zoltardata.com"
- 'Z_USERNAME': username of the account that has permission to access the resources in above app args
- 'Z_PASSWORD': password ""
"""
host = os.environ.get('Z_HOST')
username = os.environ.get('Z_USERNAME')
password = os.environ.get('Z_PASSWORD')
#
# try out non-destructive functions
#
# work with a connection
conn = ZoltarConnection(host)
conn.authenticate(username, password)
print('\n* projects')
for project in conn.projects:
print(f'- {project}, {project.id}, {project.name}')
# work with a project
project = [
project for project in conn.projects
if project.name == 'Docs Example Project'
][0]
print(f'\n* working with {project}')
print(f"- objects in {project}:\n"
f" = units: {project.units}\n"
f" = targets: {project.targets}\n"
f" = timezeros: {project.timezeros}\n"
f" = models: {project.models}")
# get the project's truth detail
print(f'\n* truth for {project}')
print(
f'- source, created_at: {project.truth_source}, {project.truth_created_at}'
)
# get the project's latest forecasts
print(f'\n* latests forecasts for {project}')
print(f'- source, created_at: {project.latest_forecasts}')
# work with a model
model = [
model for model in project.models
if model.name == 'docs forecast model'
][0]
print(f'\n* working with {model}')
print(f'- forecasts: {model.forecasts}')
# work with a forecast
forecast = model.forecasts[0]
print(f'\n* working with {forecast}')
forecast_data = forecast.data()
print(f"- data: {len(forecast_data['predictions'])} predictions"
) # 26 predictions
# work with a cdc csv file
cdc_csv_file = "tests/EW01-2011-ReichLab_kde_US_National.csv"
print(f'\n* working with a cdc csv file: {cdc_csv_file}')
with open(cdc_csv_file) as fp:
json_io_dict = json_io_dict_from_cdc_csv_file(2011, fp)
print(
f"- converted cdc data to json: {len(json_io_dict['predictions'])} predictions"
) # 154 predictions
# work with a quantile csv file
quantile_csv_file = "tests/quantile-predictions.csv"
print(f'\n* working with a quantile csv file: {quantile_csv_file}')
with open(quantile_csv_file) as fp:
json_io_dict, error_messages = \
json_io_dict_from_quantile_csv_file(fp, ['1 wk ahead cum death', '1 day ahead inc hosp'])
print(
f"- converted quantile data to json: {len(json_io_dict['predictions'])} predictions"
) # 5 predictions
# convert to a Pandas DataFrame
print(f'\n* working with a pandas data frame')
dataframe = dataframe_from_json_io_dict(forecast_data)
print(f'- dataframe: {dataframe}')
# query forecast data
print(f"\n* querying forecast data")
query = {
'targets': ['pct next week', 'cases next week'],
'types': ['point']
}
job = project.submit_query(QueryType.FORECASTS, query)
busy_poll_job(job) # does refresh()
rows = job.download_data()
print(f"- got {len(rows)} forecast rows. as a dataframe:")
print(dataframe_from_rows(rows))
# query truth data
print(f"\n* querying truth data")
query = {'targets': ['pct next week', 'cases next week']}
job = project.submit_query(QueryType.TRUTH, query)
busy_poll_job(job) # does refresh()
rows = job.download_data()
print(f"- got {len(rows)} truth rows. as a dataframe:")
print(dataframe_from_rows(rows))
#
# try out destructive functions
#
# create a sandbox project to play with, deleting the existing one if any: docs-project.json
project = [
project for project in conn.projects if project.name == 'My project'
]
project = project[0] if project else None
if project:
print(f"\n* deleting project {project}")
project.delete()
print("- deleted project")
print(f"\n* creating project")
project = create_project(
conn, "examples/docs-project.json") # "name": "My project"
print(f"- created project: {project}")
# upload truth
print(f"\n* uploading truth")
with open('tests/docs-ground-truth.csv') as csv_fp:
job = project.upload_truth_data(csv_fp)
busy_poll_job(job)
print(f"- upload truth done")
# create a model, upload a forecast, query the project, then delete it
print(f"\n* creating model")
with open("examples/example-model-config.json") as fp:
model = project.create_model(json.load(fp))
print(f"- created model: {model}")
print(f"\n* uploading forecast. pre-upload forecasts: {model.forecasts}")
with open("examples/docs-predictions.json") as fp:
json_io_dict = json.load(fp)
job = model.upload_forecast(json_io_dict, "docs-predictions.json",
"2011-10-02", "some predictions")
busy_poll_job(job)
new_forecast = job.created_forecast()
print(f"- uploaded forecast: {new_forecast}")
model.refresh()
print(f'\n* post-upload forecasts: {model.forecasts}')
print(f"\n* deleting forecast: {new_forecast}")
job = new_forecast.delete()
busy_poll_job(job)
print(f"- deleting forecast: done")
# clean up by deleting the sandbox project. NB: This will delete all of the data associated with the project without
# warning, including models and forecasts
print(f"\n* deleting project {project}")
project.delete()
print("- deleted project")
print("\n* app done!")
def connect2Zoltar():
env_user = '******'
env_pass = '******'
conn = ZoltarConnection()
conn.authenticate(os.environ.get(env_user), os.environ.get(env_pass))
return conn