def test_create_project_from_json_target_cats_format(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
project_dict = json.load(fp)
pct_next_week_target_dict = [
target_dict for target_dict in project_dict['targets']
if target_dict['name'] == 'pct next week'
][0]
project_dict['targets'] = [pct_next_week_target_dict]
# loaded cats is valid format: test that an error is not raised
try:
project = create_project_from_json(project_dict, po_user)
project.delete()
except Exception as ex:
self.fail(f"unexpected exception: {ex}")
# break cats by setting to invalid format: test that an error is raised
cats = ["not float", True, {}] # not floats
pct_next_week_target_dict['cats'] = cats
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn(
"could not convert cat to data_type. cat_str='not float'",
str(context.exception))
def test_create_project_from_json_target_range_format(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
project_dict = json.load(fp)
pct_next_week_target_dict = [
target_dict for target_dict in project_dict['targets']
if target_dict['name'] == 'pct next week'
][0]
project_dict['targets'] = [pct_next_week_target_dict]
# loaded range is valid format: test that an error is not raised
try:
project = create_project_from_json(project_dict, po_user)
project.delete()
except Exception as ex:
self.fail(f"unexpected exception: {ex}")
# break range by setting to invalid format: test that an error is raised
range_list = ["not float", True] # not floats
pct_next_week_target_dict['range'] = range_list
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("range type did not match data_type",
str(context.exception))
# test exactly two items
range_list = [1.0, 2.2, 3.3] # 3, not 2
pct_next_week_target_dict['range'] = range_list
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("range did not contain exactly two items",
str(context.exception))
def test_create_project_from_json_bad_arg(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
timezero_config = {
'timezero_date': '2017-12-01',
'data_version_date': None,
'is_season_start': True,
'season_name': 'tis the season'
}
project_dict['timezeros'] = [timezero_config]
# note: blue sky args (dict or Path) are checked elsewhere
bad_arg_exp_error = [
([1, 2],
'proj_config_file_path_or_dict was neither a dict nor a Path'),
('hi there',
'proj_config_file_path_or_dict was neither a dict nor a Path'),
(Path('forecast_app/tests/truth_data/truths-ok.csv'),
'error loading json file')
]
for bad_arg, exp_error in bad_arg_exp_error:
with self.assertRaises(RuntimeError) as context:
create_project_from_json(bad_arg, po_user)
self.assertIn(exp_error, str(context.exception))
def test_create_project_from_json_duplicate_target(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
project_dict = json.load(fp)
project_dict['targets'].append(project_dict['targets'][0])
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("found existing Target for name", str(context.exception))
def test_create_project_from_json(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
timezero_config = {
'timezero_date': '2017-12-01',
'data_version_date': None,
'is_season_start': True,
'season_name': 'tis the season'
}
project_dict['timezeros'] = [timezero_config]
project = create_project_from_json(project_dict, po_user)
# spot-check some fields
self.assertEqual(po_user, project.owner)
self.assertTrue(project.is_public)
self.assertEqual('CDC Flu challenge', project.name)
self.assertEqual(Project.WEEK_TIME_INTERVAL_TYPE,
project.time_interval_type)
self.assertEqual('Weighted ILI (%)', project.visualization_y_label)
self.assertEqual(11, project.units.count())
self.assertEqual(7, project.targets.count())
# spot-check a Unit
unit = project.units.filter(name='US National').first()
self.assertIsNotNone(unit)
# spot-check a Target
target = project.targets.filter(name='1 wk ahead').first()
self.assertEqual(Target.CONTINUOUS_TARGET_TYPE, target.type)
self.assertEqual('percent', target.unit)
self.assertTrue(target.is_step_ahead)
self.assertEqual(1, target.step_ahead_increment)
# check the TimeZero
time_zero = project.timezeros.first()
self.assertIsNotNone(time_zero)
self.assertEqual(datetime.date(2017, 12, 1), time_zero.timezero_date)
self.assertIsNone(time_zero.data_version_date)
self.assertEqual(timezero_config['is_season_start'],
time_zero.is_season_start)
self.assertEqual(timezero_config['season_name'], time_zero.season_name)
# test existing project
project.delete()
with open('forecast_app/tests/projects/cdc-project.json') as fp:
cdc_project_json = json.load(fp)
create_project_from_json(
Path('forecast_app/tests/projects/cdc-project.json'), po_user)
with self.assertRaises(RuntimeError) as context:
create_project_from_json(cdc_project_json, po_user)
self.assertIn("found existing project", str(context.exception))
def test_load_predictions_from_json_io_dict(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(is_create_super=True)
project = create_project_from_json(Path('forecast_app/tests/projects/docs-project.json'), po_user)
forecast_model = ForecastModel.objects.create(project=project, name='name', abbreviation='abbrev')
time_zero = TimeZero.objects.create(project=project, timezero_date=datetime.date(2017, 1, 1))
forecast = Forecast.objects.create(forecast_model=forecast_model, source='docs-predictions.json',
time_zero=time_zero)
# test json with no 'predictions'
with self.assertRaises(RuntimeError) as context:
load_predictions_from_json_io_dict(forecast, {}, False)
self.assertIn("json_io_dict had no 'predictions' key", str(context.exception))
# load all four types of Predictions, call Forecast.*_qs() functions. see docs-predictionsexp-rows.xlsx.
# counts from docs-predictionsexp-rows.xlsx: point: 11, named: 3, bin: 30 (3 zero prob), sample: 23
# = total rows: 67
#
# counts based on .json file:
# - 'pct next week': point: 3, named: 1 , bin: 3, sample: 5, quantile: 5 = 17
# - 'cases next week': point: 2, named: 1 , bin: 3, sample: 3, quantile: 2 = 12
# - 'season severity': point: 2, named: 0 , bin: 3, sample: 5, quantile: 0 = 10
# - 'above baseline': point: 1, named: 0 , bin: 2, sample: 6, quantile: 0 = 9
# - 'Season peak week': point: 3, named: 0 , bin: 7, sample: 4, quantile: 3 = 16
# = total rows: 64 - 2 zero prob = 62
with open('forecast_app/tests/predictions/docs-predictions.json') as fp:
json_io_dict = json.load(fp)
load_predictions_from_json_io_dict(forecast, json_io_dict, False)
self.assertEqual(62, forecast.get_num_rows())
self.assertEqual(16, forecast.bin_distribution_qs().count()) # 18 - 2 zero prob
self.assertEqual(2, forecast.named_distribution_qs().count())
self.assertEqual(11, forecast.point_prediction_qs().count())
self.assertEqual(23, forecast.sample_distribution_qs().count())
self.assertEqual(10, forecast.quantile_prediction_qs().count())
def _make_docs_project(user):
"""
Creates a project based on docs-project.json with forecasts from docs-predictions.json.
"""
found_project = Project.objects.filter(name=DOCS_PROJECT_NAME).first()
if found_project:
click.echo("* deleting previous project: {}".format(found_project))
found_project.delete()
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), user) # atomic
project.name = DOCS_PROJECT_NAME
project.save()
load_truth_data(
project, Path('forecast_app/tests/truth_data/docs-ground-truth.csv'))
forecast_model = ForecastModel.objects.create(project=project,
name='docs forecast model',
abbreviation='docs_mod')
time_zero = project.timezeros.filter(
timezero_date=datetime.date(2011, 10, 2)).first()
forecast = Forecast.objects.create(forecast_model=forecast_model,
source='docs-predictions.json',
time_zero=time_zero,
notes="a small prediction file")
with open('forecast_app/tests/predictions/docs-predictions.json') as fp:
json_io_dict_in = json.load(fp)
load_predictions_from_json_io_dict(forecast, json_io_dict_in,
False) # atomic
cache_forecast_metadata(forecast) # atomic
return project, time_zero, forecast_model, forecast
def test_target_round_trip_target_dict(self):
# test round trip: target_dict -> Target -> target_dict
# 1. target_dict -> Target
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
project_dict = json.load(fp)
input_target_dicts = project_dict['targets']
# does _validate_and_create_targets() -> model_init = {...} # required keys:
project = create_project_from_json(project_dict, po_user)
# 2. Target -> target_dict
# does target_dict() = {...} # required keys:
# note: using APIRequestFactory was the only way I could find to pass a request object. o/w you get:
# AssertionError: `HyperlinkedIdentityField` requires the request in the serializer context.
output_target_dicts = [
_target_dict_for_target(target,
APIRequestFactory().request())
for target in project.targets.all()
]
# 3. they should be equal
for target_dict in output_target_dicts: # remove 'id' and 'url' fields from TargetSerializer to ease testing
del target_dict['id']
del target_dict['url']
self.assertEqual(sorted(input_target_dicts, key=lambda _: _['name']),
sorted(output_target_dicts, key=lambda _: _['name']))
def test_prediction_dicts_to_db_rows_invalid(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(is_create_super=True)
project = create_project_from_json(Path('forecast_app/tests/projects/docs-project.json'), po_user)
forecast_model = ForecastModel.objects.create(project=project, name='name', abbreviation='abbrev')
time_zero = TimeZero.objects.create(project=project, timezero_date=datetime.date(2017, 1, 1))
forecast = Forecast.objects.create(forecast_model=forecast_model, source='docs-predictions.json',
time_zero=time_zero)
# test for invalid unit
with self.assertRaises(RuntimeError) as context:
bad_prediction_dicts = [
{"unit": "bad unit", "target": "1 wk ahead", "class": "BinCat", "prediction": {}}
]
_prediction_dicts_to_validated_db_rows(forecast, bad_prediction_dicts, False)
self.assertIn('prediction_dict referred to an undefined Unit', str(context.exception))
# test for invalid target
with self.assertRaises(RuntimeError) as context:
bad_prediction_dicts = [
{"unit": "location1", "target": "bad target", "class": "bad class", "prediction": {}}
]
_prediction_dicts_to_validated_db_rows(forecast, bad_prediction_dicts, False)
self.assertIn('prediction_dict referred to an undefined Target', str(context.exception))
# test for invalid prediction_class
with self.assertRaises(RuntimeError) as context:
bad_prediction_dicts = [
{"unit": "location1", "target": "pct next week", "class": "bad class", "prediction": {}}
]
_prediction_dicts_to_validated_db_rows(forecast, bad_prediction_dicts, False)
self.assertIn('invalid prediction_class', str(context.exception))
def test_create_project_from_json_target_types(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
# test valid types
minimal_target_dict = {
'name': 'n',
'description': 'd',
'is_step_ahead': False
} # no 'type'
target_type_int_to_required_keys_and_values = {
Target.CONTINUOUS_TARGET_TYPE:
[('unit', 'month')], # 'range' optional
Target.DISCRETE_TARGET_TYPE:
[('unit', 'month')], # 'range' optional
Target.NOMINAL_TARGET_TYPE: [('cats', ['a', 'b'])],
Target.BINARY_TARGET_TYPE: [], # binary has no required keys
Target.DATE_TARGET_TYPE: [('unit', 'month'),
('cats', ['2019-12-15', '2019-12-22'])]
}
type_int_to_name = {
type_int: type_name
for type_int, type_name in Target.TARGET_TYPE_CHOICES
}
for type_int, required_keys_and_values in target_type_int_to_required_keys_and_values.items(
):
test_target_dict = dict(minimal_target_dict) # shallow copy
project_dict['targets'] = [test_target_dict]
test_target_dict['type'] = type_int_to_name[type_int]
for required_key, value in required_keys_and_values:
test_target_dict[required_key] = value
project = create_project_from_json(project_dict, po_user)
project.delete()
# test invalid type
Project.objects.filter(name=project_dict['name']).delete()
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
first_target_dict = project_dict['targets'][0] # 'Season onset'
project_dict['targets'] = [first_target_dict]
first_target_dict['type'] = 'invalid type'
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("Invalid type_name", str(context.exception))
def test_create_project_from_json_target_required_fields(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
first_target_dict = project_dict['targets'][0] # 'Season onset'
project_dict['targets'] = [first_target_dict]
# test missing target required fields. optional keys are tested below
for field_name in ['name', 'description', 'type',
'is_step_ahead']: # required
field_value = first_target_dict[field_name]
with self.assertRaises(RuntimeError) as context:
del (first_target_dict[field_name])
create_project_from_json(project_dict, po_user)
self.assertIn("Wrong required keys in target_dict",
str(context.exception))
first_target_dict[field_name] = field_value # reset to valid
def test_load_predictions_from_json_io_dict(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), po_user)
forecast_model = ForecastModel.objects.create(project=project,
name='name',
abbreviation='abbrev')
time_zero = TimeZero.objects.create(project=project,
timezero_date=datetime.date(
2017, 1, 1))
forecast = Forecast.objects.create(forecast_model=forecast_model,
source='docs-predictions.json',
time_zero=time_zero)
# test json with no 'predictions'
with self.assertRaises(RuntimeError) as context:
load_predictions_from_json_io_dict(forecast, {},
is_validate_cats=False)
self.assertIn("json_io_dict had no 'predictions' key",
str(context.exception))
# test loading all five types of Predictions
with open(
'forecast_app/tests/predictions/docs-predictions.json') as fp:
json_io_dict = json.load(fp)
load_predictions_from_json_io_dict(forecast,
json_io_dict,
is_validate_cats=False)
# test prediction element counts match number in .json file
pred_ele_qs = forecast.pred_eles.all()
pred_data_qs = PredictionData.objects.filter(
pred_ele__forecast=forecast)
self.assertEqual(29, len(pred_ele_qs))
self.assertEqual(29, len(pred_data_qs))
# test there's a prediction element for every .json item
unit_name_to_obj = {unit.name: unit for unit in project.units.all()}
target_name_to_obj = {
target.name: target
for target in project.targets.all()
}
for pred_ele_dict in json_io_dict['predictions']:
unit = unit_name_to_obj[pred_ele_dict['unit']]
target = target_name_to_obj[pred_ele_dict['target']]
pred_class_int = PRED_CLASS_NAME_TO_INT[pred_ele_dict['class']]
data_hash = PredictionElement.hash_for_prediction_data_dict(
pred_ele_dict['prediction'])
pred_ele = pred_ele_qs.filter(pred_class=pred_class_int,
unit=unit,
target=target,
is_retract=False,
data_hash=data_hash).first()
self.assertIsNotNone(pred_ele)
self.assertIsNotNone(
pred_data_qs.filter(pred_ele=pred_ele).first())
def test_range_tuple(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
input_project_dict = json.load(fp)
project = create_project_from_json(input_project_dict, po_user)
act_range_tuples = [(target.name, target.range_tuple()) for target in project.targets.all().order_by('pk')]
self.assertEqual([('pct next week', (0.0, 100.0)),
('cases next week', (0, 100000)),
('season severity', None),
('above baseline', None),
('Season peak week', None)],
act_range_tuples)
def test_target_range_cat_validation(self):
# tests this relationship: "If `cats` are specified, then the min(`cats`) must equal the lower bound of `range`
# and max(`cats`) must be less than the upper bound of `range`."
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
input_project_dict = json.load(fp)
# test: "the min(`cats`) must equal the lower bound of `range`":
# for the "cases next week" target, change min(cats) to != min(range)
# "range": [0, 100000]
# "cats": [0, 2, 50] -> change to [1, 2, 50]
input_project_dict['targets'][1]['cats'] = [1, 2, 50]
with self.assertRaises(RuntimeError) as context:
create_project_from_json(input_project_dict, po_user)
self.assertIn("the minimum cat (1) did not equal the range's lower bound (0)", str(context.exception))
# test: "max(`cats`) must be less than the upper bound of `range`":
# for the "cases next week" target, change max(cats) to == max(range)
# "range": [0, 100000]
# "cats": [0, 2, 50] -> change to [0, 2, 100000]
input_project_dict['targets'][1]['cats'] = [0, 2, 100000]
with self.assertRaises(RuntimeError) as context:
create_project_from_json(input_project_dict, po_user)
self.assertIn("the maximum cat (100000) was not less than the range's upper bound", str(context.exception))
# also test max(cats) to > max(range)
input_project_dict['targets'][1]['cats'] = [0, 2, 100001]
with self.assertRaises(RuntimeError) as context:
create_project_from_json(input_project_dict, po_user)
self.assertIn("the maximum cat (100001) was not less than the range's upper bound ", str(context.exception))
def test_query_truth_for_project_null_rows(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), po_user)
load_truth_data(
project,
Path(
'forecast_app/tests/truth_data/docs-ground-truth-null-value.csv'
),
is_convert_na_none=True)
exp_rows = [-1] # todo xx
act_rows = list(query_truth_for_project(project, {}))
self.assertEqual(sorted(exp_rows), sorted(act_rows))
def test_create_project_from_json_target_dates_format(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
project_dict = json.load(fp)
season_peak_week_target_dict = [
target_dict for target_dict in project_dict['targets']
if target_dict['name'] == 'Season peak week'
][0]
project_dict['targets'] = [season_peak_week_target_dict]
# loaded dates are in valid 'yyyy-mm-dd' format: test that an error is not raised
try:
project = create_project_from_json(project_dict, po_user)
project.delete()
except Exception as ex:
self.fail(f"unexpected exception: {ex}")
# break dates by setting to invalid 'yyyymmdd' format: test that an error is raised
season_peak_week_target_dict['cats'] = ['2019-12-15', '20191222']
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("could not convert cat to data_type. cat_str='20191222'",
str(context.exception))
def test_load_truth_data_null_rows(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), po_user)
load_truth_data(
project,
Path(
'forecast_app/tests/truth_data/docs-ground-truth-null-value.csv'
),
is_convert_na_none=True)
exp_rows = [
(datetime.date(2011, 10, 2), 'location1', 'Season peak week', None,
None, None, datetime.date(2019, 12, 15), None),
(datetime.date(2011, 10, 2), 'location1', 'above baseline', None,
None, None, None, True),
(datetime.date(2011, 10, 2), 'location1', 'season severity', None,
None, 'moderate', None, None),
(datetime.date(2011, 10, 2), 'location1', 'cases next week', None,
None, None, None, None), # all None
(datetime.date(2011, 10, 2), 'location1', 'pct next week', None,
None, None, None, None), # all None
(datetime.date(2011, 10, 9), 'location2', 'Season peak week', None,
None, None, datetime.date(2019, 12, 29), None),
(datetime.date(2011, 10, 9), 'location2', 'above baseline', None,
None, None, None, True),
(datetime.date(2011, 10, 9), 'location2', 'season severity', None,
None, 'severe', None, None),
(datetime.date(2011, 10, 9), 'location2', 'cases next week', 3,
None, None, None, None),
(datetime.date(2011, 10, 9), 'location2', 'pct next week', None,
99.9, None, None, None),
(datetime.date(2011, 10, 16), 'location1', 'Season peak week',
None, None, None, datetime.date(2019, 12, 22), None),
(datetime.date(2011, 10, 16), 'location1', 'above baseline', None,
None, None, None, False),
(datetime.date(2011, 10, 16), 'location1', 'cases next week', 0,
None, None, None, None),
(datetime.date(2011, 10, 16), 'location1', 'pct next week', None,
0.0, None, None, None)
]
act_rows = truth_data_qs(project) \
.values_list('pred_ele__forecast__time_zero__timezero_date',
'pred_ele__unit__name', 'pred_ele__target__name',
'value_i', 'value_f', 'value_t', 'value_d', 'value_b')
self.assertEqual(sorted(exp_rows), sorted(act_rows))
def test_load_truth_data_dups(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), po_user)
load_truth_data(
project,
Path(
'forecast_app/tests/truth_data/docs-ground-truth-null-value.csv'
),
is_convert_na_none=True)
self.assertEqual(-1, truth_data_qs(project).count())
load_truth_data(
project,
Path(
'forecast_app/tests/truth_data/docs-ground-truth-null-value.csv'
),
is_convert_na_none=True)
self.assertEqual(-1, truth_data_qs(project).count())
def post(self, request, *args, **kwargs):
"""
Creates a new Project based on a project config file ala create_project_from_json(). Runs in the calling thread
and therefore blocks. POST form fields:
- request.data (required) must have a 'project_config' field containing a dict valid for
create_project_from_json(). NB: this is different from other API args in this file in that it takes all
required information as data, whereas others take their main data as a file in request.FILES, plus some
additional data in request.data.
"""
if not is_user_ok_create_project(request.user):
return HttpResponseForbidden()
elif 'project_config' not in request.data:
return JsonResponse({'error': "No 'project_config' data."}, status=status.HTTP_400_BAD_REQUEST)
try:
new_project = create_project_from_json(request.data['project_config'], request.user)
project_serializer = ProjectSerializer(new_project, context={'request': request})
return JsonResponse(project_serializer.data)
except Exception as ex:
return JsonResponse({'error': str(ex)}, status=status.HTTP_400_BAD_REQUEST)
def test_config_dict_from_project(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
input_project_dict = json.load(fp)
project = create_project_from_json(input_project_dict, po_user)
# note: using APIRequestFactory was the only way I could find to pass a request object. o/w you get:
# AssertionError: `HyperlinkedIdentityField` requires the request in the serializer context.
output_project_config = config_dict_from_project(
project,
APIRequestFactory().request())
for target_dict in output_project_config[
'targets']: # remove 'id' and 'url' fields from TargetSerializer to ease testing
del target_dict['id']
del target_dict['url']
for target_dict in output_project_config[
'timezeros']: # "" TimeZeroSerializer
del target_dict['id']
del target_dict['url']
for target_dict in output_project_config['units']: # "" UnitSerializer
del target_dict['id']
del target_dict['url']
self.assertEqual(input_project_dict, output_project_config)
def test_create_project_from_json_project_validation(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
# note: owner permissions tested by test_views_and_rest_api.py
# test missing top level fields
for field_name in [
'name', 'is_public', 'description', 'home_url', 'logo_url',
'core_data', 'time_interval_type', 'visualization_y_label',
'units', 'targets', 'timezeros'
]:
field_value = project_dict[field_name]
with self.assertRaises(RuntimeError) as context:
del (project_dict[field_name])
create_project_from_json(project_dict, po_user)
self.assertIn("Wrong keys in project_dict", str(context.exception))
project_dict[field_name] = field_value
# test units
project_dict['units'] = [{}]
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("one of the unit_dicts had no 'name' field",
str(context.exception))
# note: targets tested in test_create_project_from_json_target_validation()
# test timezero missing fields
project_dict['units'] = [{"name": "HHS Region 1"}] # reset to valid
timezero_config = {
'timezero_date': '2017-12-01',
'data_version_date': None,
'is_season_start': False
}
project_dict['timezeros'] = [timezero_config]
for field_name in [
'timezero_date', 'data_version_date', 'is_season_start'
]: # required fields
field_value = timezero_config[field_name]
with self.assertRaises(RuntimeError) as context:
del (timezero_config[field_name])
create_project_from_json(project_dict, po_user)
self.assertIn("Wrong keys in 'timezero_config'",
str(context.exception))
timezero_config[field_name] = field_value # reset to valid
# test optional 'season_name' field
timezero_config['is_season_start'] = True
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn(
'season_name not found but is required when is_season_start',
str(context.exception))
timezero_config['season_name'] = 'tis the season' # reset to valid
# test time_interval_type
project_time_interval_type = project_dict['time_interval_type']
project_dict['time_interval_type'] = "not 'week', 'biweek', or 'month'"
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("invalid 'time_interval_type'", str(context.exception))
project_dict[
'time_interval_type'] = project_time_interval_type # reset to valid
# test existing project
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
create_project_from_json(project_dict, po_user)
self.assertIn('found existing project', str(context.exception))
def test_load_predictions_from_json_io_dict_dups(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), po_user)
tz1 = TimeZero.objects.create(project=project,
timezero_date=datetime.date(2020, 10, 4))
forecast_model = ForecastModel.objects.create(project=project,
name='name',
abbreviation='abbrev')
with open(
'forecast_app/tests/predictions/docs-predictions.json') as fp:
json_io_dict = json.load(fp)
pred_dicts = json_io_dict[
'predictions'] # get some prediction elements to work with (29)
# per https://stackoverflow.com/questions/1937622/convert-date-to-datetime-in-python/1937636 :
f1 = Forecast.objects.create(forecast_model=forecast_model,
source='f1',
time_zero=tz1,
issued_at=datetime.datetime.combine(
tz1.timezero_date,
datetime.time(),
tzinfo=datetime.timezone.utc))
load_predictions_from_json_io_dict(f1, {
'meta': {},
'predictions': pred_dicts[:-2]
}) # all but last 2 PEs
# case: load the just-loaded file into a separate timezero -> should load all rows (duplicates are only within
# the same timezero)
tz2 = project.timezeros.filter(
timezero_date=datetime.date(2011, 10, 9)).first()
f2 = Forecast.objects.create(forecast_model=forecast_model,
time_zero=tz2)
load_predictions_from_json_io_dict(
f2,
{
'meta': {},
'predictions': pred_dicts[:-1]
}, # all but last PE
is_validate_cats=False)
self.assertEqual(27, f1.pred_eles.count())
self.assertEqual(28, f2.pred_eles.count())
self.assertEqual(27 + 28,
project.num_pred_ele_rows_all_models(is_oracle=False))
# case: load the same predictions into a different version -> none should load (they're all duplicates)
f1.issued_at -= datetime.timedelta(days=1)
f1.save()
f3 = Forecast.objects.create(forecast_model=forecast_model,
time_zero=tz1)
load_predictions_from_json_io_dict(f3,
json_io_dict,
is_validate_cats=False)
self.assertEqual(27, f1.pred_eles.count())
self.assertEqual(28, f2.pred_eles.count())
self.assertEqual(2, f3.pred_eles.count()) # 2 were new (non-dup)
self.assertEqual(27 + 28 + 2,
project.num_pred_ele_rows_all_models(is_oracle=False))
# case: load the same file, but change one multi-row prediction (a sample) to have partial duplication
f3.issued_at -= datetime.timedelta(days=2)
f3.save()
quantile_pred_dict = [
pred_dict for pred_dict in json_io_dict['predictions']
if (pred_dict['unit'] == 'location2') and (
pred_dict['target'] == 'pct next week') and (
pred_dict['class'] == 'quantile')
][0]
# original: {"quantile": [0.025, 0.25, 0.5, 0.75, 0.975 ],
# "value": [1.0, 2.2, 2.2, 5.0, 50.0 ]}
quantile_pred_dict['prediction']['value'][0] = 2.2 # was 1.0
f4 = Forecast.objects.create(forecast_model=forecast_model,
time_zero=tz1)
load_predictions_from_json_io_dict(f4,
json_io_dict,
is_validate_cats=False)
self.assertEqual(1, f4.pred_eles.count())
self.assertEqual(27 + 28 + 2 + 1,
project.num_pred_ele_rows_all_models(is_oracle=False))
def test_group_targets(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
# case: target names with step_ahead_increment at start of name
project = create_project_from_json(
Path('forecast_app/tests/projects/COVID-19_Forecasts-config.json'),
po_user)
grouped_targets = group_targets(project.targets.all())
# group 1: "_ day ahead cum death" | 0 day ahead cum death, 1 day ahead cum death, ..., 130 day ahead cum death
# group 2: "_ day ahead inc death" | 0, 1, ..., 130
# group 3: "_ day ahead inc hosp" | 0, 1, ..., 130
# group 4: "_ wk ahead cum death" | 1 wk ahead cum death, 2 wk ahead cum death, ..., 20 wk ahead cum death
# group 5: "_ wk ahead inc death" | 1, 2, ..., 20
# group 6: "_ wk ahead inc case" | 1 wk ahead inc case, 2 wk ahead inc case, ..., 8 wk ahead inc case
self.assertEqual(6, len(grouped_targets))
self.assertEqual(
{
'day ahead inc hosp', 'day ahead inc death',
'day ahead cum death', 'wk ahead inc death',
'wk ahead cum death', 'wk ahead inc case'
}, set(grouped_targets.keys()))
self.assertEqual(131, len(grouped_targets['day ahead inc hosp']))
self.assertEqual(131, len(grouped_targets['day ahead inc death']))
self.assertEqual(131, len(grouped_targets['day ahead cum death']))
self.assertEqual(20, len(grouped_targets['wk ahead inc death']))
self.assertEqual(20, len(grouped_targets['wk ahead cum death']))
self.assertEqual(8, len(grouped_targets['wk ahead inc case']))
# case: mix of target names with step_ahead_increment at start of name, and others
project = create_project_from_json(
Path('forecast_app/tests/projects/cdc-project.json'), po_user)
grouped_targets = group_targets(project.targets.all())
# group 1: "Season onset"
# group 2: "Season peak week"
# group 3: "Season peak percentage"
# group 4: "x wk ahead" | 1 wk ahead, 2 wk ahead, 3 wk ahead, 4 wk ahead
self.assertEqual(4, len(grouped_targets))
self.assertEqual(
{
'Season onset', 'Season peak week', 'Season peak percentage',
'wk ahead'
}, set(grouped_targets.keys()))
self.assertEqual(4, len(grouped_targets['wk ahead']))
# case: target names with step_ahead_increment inside the name (i.e., not at start)
project = Project.objects.create()
for step_ahead_increment in range(2):
target_init = {
'project': project,
'name': f'wk {step_ahead_increment} ahead',
'type': Target.CONTINUOUS_TARGET_TYPE,
'is_step_ahead': True,
'step_ahead_increment': step_ahead_increment,
'unit': 'cases'
}
Target.objects.create(**target_init)
grouped_targets = group_targets(project.targets.all())
# group 1: 'wk x ahead'
self.assertEqual(1, len(grouped_targets))
self.assertEqual({'wk ahead'}, set(grouped_targets.keys()))
self.assertEqual(2, len(grouped_targets['wk ahead']))
# case: targets with no word boundaries
project = create_project_from_json(
Path('forecast_app/tests/projects/thai-project.json'), po_user)
grouped_targets = group_targets(project.targets.all())
# group 1: "x_biweek_ahead" | 1_biweek_ahead, 2_biweek_ahead, 3_biweek_ahead, 4_biweek_ahead, 5_biweek_ahead
self.assertEqual(1, len(grouped_targets))
self.assertEqual({'biweek ahead'}, set(grouped_targets.keys()))
self.assertEqual(5, len(grouped_targets['biweek ahead']))
# case: similar names, different types
project = Project.objects.create()
for step_ahead_increment, target_type in [
(0, Target.CONTINUOUS_TARGET_TYPE),
(1, Target.DISCRETE_TARGET_TYPE)
]:
target_init = {
'project': project,
'name': f'wk {step_ahead_increment} ahead',
'type': target_type,
'is_step_ahead': True,
'step_ahead_increment': step_ahead_increment,
'unit': 'cases'
}
Target.objects.create(**target_init)
grouped_targets = group_targets(project.targets.all())
# group 1: 'wk ahead' (discrete)
# group 2: 'wk ahead 2' (continuous)
self.assertEqual(2, len(grouped_targets))
self.assertEqual({'wk ahead', 'wk ahead 2'},
set(grouped_targets.keys()))
# case: similar names, different units
project = Project.objects.create()
for step_ahead_increment, target_unit in [(0, 'unit 1'),
(1, 'unit 2')]:
target_init = {
'project': project,
'name': f'wk {step_ahead_increment} ahead',
'type': Target.CONTINUOUS_TARGET_TYPE,
'is_step_ahead': True,
'step_ahead_increment': step_ahead_increment,
'unit': target_unit
}
Target.objects.create(**target_init)
grouped_targets = group_targets(project.targets.all())
# group 1: 'wk ahead' ('unit 2')
# group 2: 'wk ahead 2' ('unit 1')
self.assertEqual(2, len(grouped_targets))
self.assertEqual({'wk ahead', 'wk ahead 2'},
set(grouped_targets.keys()))
def test_target_range_cats_lwr_relationship(self):
# test this relationship: "if `range` had been specified as [0, 100] in addition to the above `cats`, then the
# final bin would be [2.2, 100]."
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
input_project_dict = json.load(fp)
create_project_from_json(input_project_dict, po_user)
# "pct next week":
# "range": [0.0, 100.0] -> TargetRange: 2 value_f
# "cats": [0.0, 1.0, 1.1, 2.0, 2.2, 3.0, 3.3, 5.0, 10.0, 50.0] -> TargetCat: 10 value_f
# -> TargetLwr: 10: lwr/upper: [(0.0, 1.0), (1.0, 1.1), (1.1, 2.0), (2.0, 2.2), (2.2, 3.0), (3.0, 3.3),
# (3.3, 5.0), (5.0, 10.0), (10.0, 50.0), (50.0, 100.0)]
pct_next_week_target = Target.objects.filter(name='pct next week').first()
ranges_qs = pct_next_week_target.ranges.all() \
.order_by('value_f') \
.values_list('target__name', 'value_i', 'value_f')
self.assertEqual([('pct next week', None, 0.0), ('pct next week', None, 100.0)], list(ranges_qs))
cats_qs = pct_next_week_target.cats.all() \
.order_by('cat_f') \
.values_list('target__name', 'cat_i', 'cat_f', 'cat_t', 'cat_d', 'cat_b')
exp_cats = [('pct next week', None, 0.0, None, None, None), ('pct next week', None, 1.0, None, None, None),
('pct next week', None, 1.1, None, None, None), ('pct next week', None, 2.0, None, None, None),
('pct next week', None, 2.2, None, None, None), ('pct next week', None, 3.0, None, None, None),
('pct next week', None, 3.3, None, None, None), ('pct next week', None, 5.0, None, None, None),
('pct next week', None, 10.0, None, None, None), ('pct next week', None, 50.0, None, None, None)]
self.assertEqual(exp_cats, list(cats_qs))
lwrs_qs = pct_next_week_target.lwrs.all() \
.order_by('lwr') \
.values_list('target__name', 'lwr', 'upper')
exp_lwrs = [('pct next week', 0.0, 1.0), ('pct next week', 1.0, 1.1), ('pct next week', 1.1, 2.0),
('pct next week', 2.0, 2.2), ('pct next week', 2.2, 3.0), ('pct next week', 3.0, 3.3),
('pct next week', 3.3, 5.0), ('pct next week', 5.0, 10.0), ('pct next week', 10.0, 50.0),
('pct next week', 50.0, 100.0), ('pct next week', 100.0, float('inf'))]
self.assertEqual(exp_lwrs, list(lwrs_qs))
# "cases next week":
# "range": [0, 100000] -> TargetRange: 2 value_i
# "cats": [0, 2, 50] -> TargetCat: 3 value_i
# -> TargetLwr: 3: lwr/upper: [(0, 2), (2, 50), (50, 100000)]
cases_next_week_target = Target.objects.filter(name='cases next week').first()
ranges_qs = cases_next_week_target.ranges.all() \
.order_by('value_i') \
.values_list('target__name', 'value_i', 'value_f')
self.assertEqual([('cases next week', 0, None), ('cases next week', 100000, None)], list(ranges_qs))
cats_qs = cases_next_week_target.cats.all() \
.order_by('cat_i') \
.values_list('target__name', 'cat_i', 'cat_f', 'cat_t', 'cat_d', 'cat_b')
exp_cats = [('cases next week', 0, None, None, None, None),
('cases next week', 2, None, None, None, None),
('cases next week', 50, None, None, None, None)]
self.assertEqual(exp_cats, list(cats_qs))
lwrs_qs = cases_next_week_target.lwrs.all() \
.order_by('lwr') \
.values_list('target__name', 'lwr', 'upper')
exp_lwrs = [('cases next week', 0.0, 2.0), ('cases next week', 2.0, 50.0), ('cases next week', 50.0, 100000.0),
('cases next week', 100000.0, float('inf'))]
self.assertEqual(exp_lwrs, list(lwrs_qs))
def test_create_project_from_json_cats_lws_ranges_created(self):
# verify that 'list' TargetCat, TargetLwr, and TargetRange instances are created.
# docs-project.json contains examples of all five target types
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
project_dict = json.load(fp)
project = create_project_from_json(project_dict, po_user)
# test 'pct next week' target. continuous, with range and cats (w/lwrs)
target = project.targets.filter(name='pct next week').first()
self.assertEqual(Target.CONTINUOUS_TARGET_TYPE, target.type)
self.assertEqual('percent', target.unit)
self.assertTrue(target.is_step_ahead)
self.assertEqual(1, target.step_ahead_increment)
ranges = target.ranges.all().order_by('value_f')
self.assertEqual(2, len(ranges))
self.assertEqual([(None, 0.0), (None, 100.0)],
list(ranges.values_list('value_i', 'value_f')))
cats = target.cats.all().order_by('cat_f')
self.assertEqual(10, len(cats))
self.assertEqual([(None, 0.0, None, None, None),
(None, 1.0, None, None, None),
(None, 1.1, None, None, None),
(None, 2.0, None, None, None),
(None, 2.2, None, None, None),
(None, 3.0, None, None, None),
(None, 3.3, None, None, None),
(None, 5.0, None, None, None),
(None, 10.0, None, None, None),
(None, 50.0, None, None, None)],
list(
cats.values_list('cat_i', 'cat_f', 'cat_t',
'cat_d', 'cat_b')))
lwrs = target.lwrs.all().order_by('lwr', 'upper')
self.assertEqual(11, len(lwrs))
self.assertEqual([(0.0, 1.0), (1.0, 1.1), (1.1, 2.0), (2.0, 2.2),
(2.2, 3.0), (3.0, 3.3), (3.3, 5.0), (5.0, 10.0),
(10.0, 50.0), (50.0, 100.0), (100.0, float('inf'))],
list(lwrs.values_list('lwr', 'upper')))
# test 'cases next week' target. discrete, with range
target = project.targets.filter(name='cases next week').first()
ranges = target.ranges.all().order_by('value_i')
self.assertEqual(2, len(ranges))
self.assertEqual([(0, None), (100000, None)],
list(ranges.values_list('value_i', 'value_f')))
# test 'season severity' target. nominal, with cats
target = project.targets.filter(name='season severity').first()
cats = target.cats.all().order_by('cat_t')
self.assertEqual(4, len(cats))
self.assertEqual([(None, None, 'high', None, None),
(None, None, 'mild', None, None),
(None, None, 'moderate', None, None),
(None, None, 'severe', None, None)],
list(
cats.values_list('cat_i', 'cat_f', 'cat_t',
'cat_d', 'cat_b')))
# test 'above baseline' target. binary, with two implicit boolean cats created behind-the-scenes
target = project.targets.filter(name='above baseline').first()
cats = target.cats.all().order_by('cat_b')
self.assertEqual(2, len(cats))
self.assertEqual([(None, None, None, None, False),
(None, None, None, None, True)],
list(
cats.values_list('cat_i', 'cat_f', 'cat_t',
'cat_d', 'cat_b')))
# test 'Season peak week' target. date, with dates as cats
target = project.targets.filter(name='Season peak week').first()
dates = target.cats.all().order_by('cat_d') # date
self.assertEqual(4, len(dates))
self.assertEqual([
datetime.date(2019, 12, 15),
datetime.date(2019, 12, 22),
datetime.date(2019, 12, 29),
datetime.date(2020, 1, 5)
], list(dates.values_list('cat_d', flat=True)))
def test_as_of_versions(self):
# tests the case in [Add forecast versioning](https://github.com/reichlab/forecast-repository/issues/273):
#
# Here's an example database with versions (header is timezeros, rows are forecast `issue_date`s). Each forecast
# only has one point prediction:
#
# +-----+-----+-----+
# |10/2 |10/9 |10/16|
# |tz1 |tz2 |tz3 |
# +=====+=====+=====+
# |10/2 | | |
# |f1 | - | - | 2.1
# +-----+-----+-----+
# | | |10/17|
# |- | - |f2 | 2.0
# +-----+-----+-----+
# |10/20|10/20| |
# |f3 | f4 | - | 3.567 | 10
# +-----+-----+-----+
#
# Here are some `as_of` examples (which forecast version would be used as of that date):
#
# +-----+----+----+----+
# |as_of|tz1 |tz2 |tz3 |
# +-----+----+----+----+
# |10/1 | - | - | - |
# |10/3 | f1 | - | - |
# |10/18| f1 | - | f2 |
# |10/20| f3 | f4 | f2 |
# |10/21| f3 | f4 | f2 |
# +-----+----+----+----+
# set up database
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'),
po_user) # atomic
forecast_model = ForecastModel.objects.create(
project=project,
name='docs forecast model',
abbreviation='docs_mod')
tz1 = project.timezeros.filter(
timezero_date=datetime.date(2011, 10, 2)).first()
tz2 = project.timezeros.filter(
timezero_date=datetime.date(2011, 10, 9)).first()
tz3 = project.timezeros.filter(
timezero_date=datetime.date(2011, 10, 16)).first()
f1 = Forecast.objects.create(forecast_model=forecast_model,
source='f1',
time_zero=tz1)
json_io_dict = {
"predictions": [{
"unit": "location1",
"target": "pct next week",
"class": "point",
"prediction": {
"value": 2.1
}
}]
}
load_predictions_from_json_io_dict(f1, json_io_dict, False)
f1.issue_date = tz1.timezero_date
f1.save()
f2 = Forecast.objects.create(forecast_model=forecast_model,
source='f2',
time_zero=tz3)
json_io_dict = {
"predictions": [{
"unit": "location2",
"target": "pct next week",
"class": "point",
"prediction": {
"value": 2.0
}
}]
}
load_predictions_from_json_io_dict(f2, json_io_dict, False)
f2.issue_date = tz3.timezero_date + datetime.timedelta(days=1)
f2.save()
f3 = Forecast.objects.create(forecast_model=forecast_model,
source='f3',
time_zero=tz1)
json_io_dict = {
"predictions": [{
"unit": "location3",
"target": "pct next week",
"class": "point",
"prediction": {
"value": 3.567
}
}]
}
load_predictions_from_json_io_dict(f3, json_io_dict, False)
f3.issue_date = tz1.timezero_date + datetime.timedelta(days=18)
f3.save()
f4 = Forecast.objects.create(forecast_model=forecast_model,
source='f4',
time_zero=tz2)
json_io_dict = {
"predictions": [{
"unit": "location3",
"target": "cases next week",
"class": "point",
"prediction": {
"value": 10
}
}]
}
load_predictions_from_json_io_dict(f4, json_io_dict, False)
f4.issue_date = f3.issue_date
f4.save()
# case: default (no `as_of`): no f1 (f3 is newer)
exp_rows = [
['2011-10-16', 'location2', 'pct next week', 'point', 2.0],
['2011-10-02', 'location3', 'pct next week', 'point', 3.567],
['2011-10-09', 'location3', 'cases next week', 'point', 10]
]
act_rows = list(query_forecasts_for_project(project, {}))
act_rows = [row[1:2] + row[3:7] for row in act_rows[1:]
] # 'timezero', 'unit', 'target', 'class', 'value'
self.assertEqual(exp_rows, act_rows)
# case: 10/20: same as default
act_rows = list(
query_forecasts_for_project(project, {'as_of': '2011-10-20'}))
act_rows = [row[1:2] + row[3:7] for row in act_rows[1:]
] # 'timezero', 'unit', 'target', 'class', 'value'
self.assertEqual(exp_rows, act_rows)
# case: 10/21: same as default
act_rows = list(
query_forecasts_for_project(project, {'as_of': '2011-10-21'}))
act_rows = [row[1:2] + row[3:7] for row in act_rows[1:]
] # 'timezero', 'unit', 'target', 'class', 'value'
self.assertEqual(exp_rows, act_rows)
# case: 10/1: none
exp_rows = []
act_rows = list(
query_forecasts_for_project(project, {'as_of': '2011-10-01'}))
act_rows = [row[1:2] + row[3:7] for row in act_rows[1:]
] # 'timezero', 'unit', 'target', 'class', 'value'
self.assertEqual(exp_rows, act_rows)
# case: 10/3: just f1
exp_rows = [['2011-10-02', 'location1', 'pct next week', 'point', 2.1]]
act_rows = list(
query_forecasts_for_project(project, {'as_of': '2011-10-03'}))
act_rows = [row[1:2] + row[3:7] for row in act_rows[1:]
] # 'timezero', 'unit', 'target', 'class', 'value'
self.assertEqual(exp_rows, act_rows)
# case: 10/18: f1 and f2
exp_rows = [['2011-10-02', 'location1', 'pct next week', 'point', 2.1],
['2011-10-16', 'location2', 'pct next week', 'point', 2.0]]
act_rows = list(
query_forecasts_for_project(project, {'as_of': '2011-10-18'}))
act_rows = [row[1:2] + row[3:7] for row in act_rows[1:]
] # 'timezero', 'unit', 'target', 'class', 'value'
self.assertEqual(exp_rows, act_rows)
def test_json_io_dict_from_forecast(self):
# tests that the json_io_dict_from_forecast()'s output order for SampleDistributions is preserved
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), po_user)
forecast_model = ForecastModel.objects.create(project=project,
name='name',
abbreviation='abbrev')
time_zero = TimeZero.objects.create(project=project,
timezero_date=datetime.date(
2017, 1, 1))
forecast = Forecast.objects.create(forecast_model=forecast_model,
source='docs-predictions.json',
time_zero=time_zero)
with open(
'forecast_app/tests/predictions/docs-predictions.json') as fp:
json_io_dict_in = json.load(fp)
load_predictions_from_json_io_dict(forecast,
json_io_dict_in,
is_validate_cats=False)
json_io_dict_out = json_io_dict_from_forecast(
forecast,
APIRequestFactory().request())
# test round trip. ignore meta, but spot-check it first
out_meta = json_io_dict_out['meta']
self.assertEqual({'targets', 'forecast', 'units'},
set(out_meta.keys()))
self.assertEqual(
{
'cats', 'unit', 'name', 'is_step_ahead', 'type', 'description',
'id', 'url'
}, set(out_meta['targets'][0].keys()))
self.assertEqual(
{
'time_zero', 'forecast_model', 'created_at', 'issued_at',
'notes', 'forecast_data', 'source', 'id', 'url'
}, set(out_meta['forecast'].keys()))
self.assertEqual({'id', 'name', 'url'},
set(out_meta['units'][0].keys()))
self.assertIsInstance(out_meta['forecast']['time_zero'],
dict) # test that time_zero is expanded, not URL
del (json_io_dict_in['meta'])
del (json_io_dict_out['meta'])
json_io_dict_in['predictions'].sort(
key=lambda _: (_['unit'], _['target'], _['class']))
json_io_dict_out['predictions'].sort(
key=lambda _: (_['unit'], _['target'], _['class']))
self.assertEqual(json_io_dict_in, json_io_dict_out)
# spot-check some sample predictions
sample_pred_dict = [
pred_dict for pred_dict in json_io_dict_out['predictions']
if (pred_dict['unit'] == 'location3') and (
pred_dict['target'] == 'pct next week') and (
pred_dict['class'] == 'sample')
][0]
self.assertEqual([2.3, 6.5, 0.0, 10.0234, 0.0001],
sample_pred_dict['prediction']['sample'])
sample_pred_dict = [
pred_dict for pred_dict in json_io_dict_out['predictions']
if (pred_dict['unit'] == 'location2') and (
pred_dict['target'] == 'season severity') and (
pred_dict['class'] == 'sample')
][0]
self.assertEqual(['moderate', 'severe', 'high', 'moderate', 'mild'],
sample_pred_dict['prediction']['sample'])
sample_pred_dict = [
pred_dict for pred_dict in json_io_dict_out['predictions']
if (pred_dict['unit'] == 'location1') and (
pred_dict['target'] == 'Season peak week') and (
pred_dict['class'] == 'sample')
][0]
self.assertEqual(['2020-01-05', '2019-12-15'],
sample_pred_dict['prediction']['sample'])
# spot-check some quantile predictions
quantile_pred_dict = [
pred_dict for pred_dict in json_io_dict_out['predictions']
if (pred_dict['unit'] == 'location2') and (
pred_dict['target'] == 'pct next week') and (
pred_dict['class'] == 'quantile')
][0]
self.assertEqual([0.025, 0.25, 0.5, 0.75, 0.975],
quantile_pred_dict['prediction']['quantile'])
self.assertEqual([1.0, 2.2, 2.2, 5.0, 50.0],
quantile_pred_dict['prediction']['value'])
quantile_pred_dict = [
pred_dict for pred_dict in json_io_dict_out['predictions']
if (pred_dict['unit'] == 'location2') and (
pred_dict['target'] == 'Season peak week') and (
pred_dict['class'] == 'quantile')
][0]
self.assertEqual([0.5, 0.75, 0.975],
quantile_pred_dict['prediction']['quantile'])
self.assertEqual(["2019-12-22", "2019-12-29", "2020-01-05"],
quantile_pred_dict['prediction']['value'])
def project_config_diff(config_dict_1, config_dict_2):
"""
Analyzes and returns the differences between the two project configuration dicts, specifically the changes that were
made to config_dict_1 to result in config_dict_2. Here are the kinds of diffs:
Project edits:
- editable fields: 'name', 'is_public', 'description', 'home_url', 'logo_url', 'core_data', 'time_interval_type',
'visualization_y_label'
- 'units': add, remove
- 'timezeros': add, remove, edit
- 'targets': add, remove, edit
Unit edits: fields:
- 'name': the Unit's pk. therefore editing this field effectively removes the existing Unit and adds a new
one to replace it
TimeZero edits: fields:
- 'timezero_date': the TimeZero's pk. therefore editing this field effectively removes the existing TimeZero and
adds a new one to replace it
- editable fields: 'data_version_date', 'is_season_start', 'season_name'
Target edits: fields:
- 'name': the Target's pk. therefore editing this field effectively removes the existing Target and adds a new one
to replace it
- editable fields: 'description', 'is_step_ahead', 'step_ahead_increment', 'unit'
- 'type': Target type cannot be edited because it might invalidate existing forecasts. therefore editing this field
effectively removes the existing Target and adds a new one to replace it
- 'range': similarly cannot be edited due to possible forecast invalidation
- 'cats': ""
:param config_dict_1: as returned by config_dict_from_project(). treated as the "from" dict
:param config_dict_2: "". treated as the "to" dict
:return: a list of Change objects that 'move' the state of config_dict_1 to config_dict_2. aka a
"project config diff". list order is non-deterministic
"""
changes = [] # return value. filled next. a list of Changes
# validate inputs (ensures expected fields are present)
create_project_from_json(config_dict_1, None, is_validate_only=True)
create_project_from_json(config_dict_2, None, is_validate_only=True)
# check project field edits
for field_name in [
'name', 'is_public', 'description', 'home_url', 'logo_url',
'core_data', 'time_interval_type', 'visualization_y_label'
]:
if config_dict_1[field_name] != config_dict_2[field_name]:
changes.append(
Change(ObjectType.PROJECT, None, ChangeType.FIELD_EDITED,
field_name, config_dict_2))
# check for units added or removed
unit_names_1 = {unit_dict['name'] for unit_dict in config_dict_1['units']}
unit_names_2 = {unit_dict['name'] for unit_dict in config_dict_2['units']}
removed_loc_names = unit_names_1 - unit_names_2
added_loc_names = unit_names_2 - unit_names_1
changes.extend([
Change(ObjectType.UNIT, name, ChangeType.OBJ_REMOVED, None, None)
for name in removed_loc_names
])
changes.extend([
Change(ObjectType.UNIT, unit_dict['name'], ChangeType.OBJ_ADDED, None,
unit_dict) for unit_dict in config_dict_2['units']
if unit_dict['name'] in added_loc_names
])
# check for timezeros added or removed
timezero_dates_1 = {
timezero_dict['timezero_date']
for timezero_dict in config_dict_1['timezeros']
}
timezero_dates_2 = {
timezero_dict['timezero_date']
for timezero_dict in config_dict_2['timezeros']
}
removed_tz_dates = timezero_dates_1 - timezero_dates_2
added_tz_dates = timezero_dates_2 - timezero_dates_1
changes.extend([
Change(ObjectType.TIMEZERO, name, ChangeType.OBJ_REMOVED, None, None)
for name in removed_tz_dates
])
changes.extend([
Change(ObjectType.TIMEZERO, tz_dict['timezero_date'],
ChangeType.OBJ_ADDED, None, tz_dict)
for tz_dict in config_dict_2['timezeros']
if tz_dict['timezero_date'] in added_tz_dates
])
# check for timezero field edits
tz_date_1_to_dict = {
timezero_dict['timezero_date']: timezero_dict
for timezero_dict in config_dict_1['timezeros']
}
tz_date_2_to_dict = {
timezero_dict['timezero_date']: timezero_dict
for timezero_dict in config_dict_2['timezeros']
}
for timezero_date in timezero_dates_1 & timezero_dates_2: # timezero_dates_both
for field_name in [
'data_version_date', 'is_season_start', 'season_name'
]: # season_name is only optional field
if (field_name in tz_date_1_to_dict[timezero_date]) and \
(field_name not in tz_date_2_to_dict[timezero_date]):
# field_name removed
changes.append(
Change(ObjectType.TIMEZERO, timezero_date,
ChangeType.FIELD_REMOVED, field_name, None))
elif (field_name not in tz_date_1_to_dict[timezero_date]) and \
(field_name in tz_date_2_to_dict[timezero_date]):
# field_name added
changes.append(
Change(ObjectType.TIMEZERO, timezero_date,
ChangeType.FIELD_ADDED, field_name,
tz_date_2_to_dict[timezero_date]))
# this test for `!= ''` matches this one below: "NB: here we convert '' to None to avoid errors like"
elif (field_name in tz_date_1_to_dict[timezero_date]) and \
(field_name in tz_date_2_to_dict[timezero_date]) and \
(tz_date_2_to_dict[timezero_date][field_name] != '') and \
(tz_date_1_to_dict[timezero_date][field_name] != tz_date_2_to_dict[timezero_date][field_name]):
# field_name edited
changes.append(
Change(ObjectType.TIMEZERO, timezero_date,
ChangeType.FIELD_EDITED, field_name,
tz_date_2_to_dict[timezero_date]))
# check for targets added or removed
target_names_1 = {
target_dict['name']
for target_dict in config_dict_1['targets']
}
target_names_2 = {
target_dict['name']
for target_dict in config_dict_2['targets']
}
removed_target_names = target_names_1 - target_names_2
added_target_names = target_names_2 - target_names_1
changes.extend([
Change(ObjectType.TARGET, name, ChangeType.OBJ_REMOVED, None, None)
for name in removed_target_names
])
changes.extend([
Change(ObjectType.TARGET, target_dict['name'], ChangeType.OBJ_ADDED,
None, target_dict) for target_dict in config_dict_2['targets']
if target_dict['name'] in added_target_names
])
# check for target field edits. as noted above, editing some fields imply entire target replacement (remove and then
# add)
targ_name_1_to_dict = {
target_dict['name']: target_dict
for target_dict in config_dict_1['targets']
}
targ_name_2_to_dict = {
target_dict['name']: target_dict
for target_dict in config_dict_2['targets']
}
editable_fields = [
'description', 'is_step_ahead', 'step_ahead_increment', 'unit'
]
non_editable_fields = ['type', 'range', 'cats']
for target_name in target_names_1 & target_names_2: # target_names_both
for field_name in editable_fields + non_editable_fields:
if (field_name in targ_name_1_to_dict[target_name]) and \
(field_name not in targ_name_2_to_dict[target_name]):
# field_name removed
if field_name in non_editable_fields:
changes.append(
Change(ObjectType.TARGET, target_name,
ChangeType.OBJ_REMOVED, None, None))
changes.append(
Change(ObjectType.TARGET, target_name,
ChangeType.OBJ_ADDED, None,
targ_name_2_to_dict[target_name])
) # use 2nd dict in case other changes
else:
changes.append(
Change(ObjectType.TARGET, target_name,
ChangeType.FIELD_REMOVED, field_name, None))
elif (field_name not in targ_name_1_to_dict[target_name]) and \
(field_name in targ_name_2_to_dict[target_name]):
# field_name added
if field_name in non_editable_fields:
changes.append(
Change(ObjectType.TARGET, target_name,
ChangeType.OBJ_REMOVED, None, None))
changes.append(
Change(ObjectType.TARGET, target_name,
ChangeType.OBJ_ADDED, None,
targ_name_2_to_dict[target_name])
) # use 2nd dict in case other changes
else:
changes.append(
Change(ObjectType.TARGET, target_name,
ChangeType.FIELD_ADDED, field_name,
targ_name_2_to_dict[target_name]))
elif (field_name in targ_name_1_to_dict[target_name]) and \
(field_name in targ_name_2_to_dict[target_name]) and \
(targ_name_1_to_dict[target_name][field_name] != targ_name_2_to_dict[target_name][field_name]):
# field_name edited
if field_name in non_editable_fields:
changes.append(
Change(ObjectType.TARGET, target_name,
ChangeType.OBJ_REMOVED, None, None))
changes.append(
Change(ObjectType.TARGET, target_name,
ChangeType.OBJ_ADDED, None,
targ_name_2_to_dict[target_name])
) # use 2nd dict in case other changes
else:
changes.append(
Change(ObjectType.TARGET, target_name,
ChangeType.FIELD_EDITED, field_name,
targ_name_2_to_dict[target_name]))
# done
return changes
def test_load_predictions_from_json_io_dict_phase_1(self):
# tests pass 1/2 of load_predictions_from_json_io_dict(). NB: implicitly covers test_hash_for_prediction_dict()
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), po_user)
forecast_model = ForecastModel.objects.create(project=project,
name='name',
abbreviation='abbrev')
time_zero = TimeZero.objects.create(project=project,
timezero_date=datetime.date(
2017, 1, 1))
forecast = Forecast.objects.create(forecast_model=forecast_model,
source='docs-predictions.json',
time_zero=time_zero)
with open(
'forecast_app/tests/predictions/docs-predictions.json') as fp:
json_io_dict = json.load(fp)
load_predictions_from_json_io_dict(forecast,
json_io_dict,
is_validate_cats=False)
# test PredictionElement.forecast and is_retract
self.assertEqual(29, forecast.pred_eles.count())
self.assertEqual(
0,
PredictionElement.objects.filter(is_retract=True).count())
exp_rows = [
('point', 'location1', 'pct next week',
'2c343e1ea37e8b493c219066a8664276'),
('named', 'location1', 'pct next week',
'58a7f8487958446d57333b262aaa8271'),
('point', 'location2', 'pct next week',
'2b9db448ae1a3b7065ffee67d4857268'),
('bin', 'location2', 'pct next week',
'7d1485af48de540dbcd954ee5cba51cb'),
('quantile', 'location2', 'pct next week',
'0d3698e4e39456b8e36c750d73bb6870'),
('point', 'location3', 'pct next week',
'5d321ea39f0af08cb3f40a58fa7c54d4'),
('sample', 'location3', 'pct next week',
'0b431a76d5ad343981944c4b0792d738'),
('named', 'location1', 'cases next week',
'845e3d041b6be23a381b6afd263fb113'),
('point', 'location2', 'cases next week',
'2ed5d7d59eb10044644ab28a1b292efb'),
('sample', 'location2', 'cases next week',
'74135c30ddfd5427c8b1e86b2989a642'),
('point', 'location3', 'cases next week',
'a6ff82cc0637f67254df41352e1c00f9'),
('bin', 'location3', 'cases next week',
'a74ea3f2472e0aec511eb1f604282220'),
('quantile', 'location3', 'cases next week',
'838e6e3f77075f69eef3bb3d7bcdffdc'),
('point', 'location1', 'season severity',
'bc55989f596fd157ccc6e3279b1f694a'),
('bin', 'location1', 'season severity',
'ac263a19694da72f65e903c2ec2000d1'),
('point', 'location2', 'season severity',
'ec5add7ea7a8abf3e68e9570d0b73898'),
('sample', 'location2', 'season severity',
'51d07bda3e8a39da714f5767d93704ff'),
('point', 'location1', 'above baseline',
'19d0e94bc24114abfa0d07ca41b8b3bf'),
('bin', 'location2', 'above baseline',
'1b98c3c7b5b09d3ba0ea43566d5e9d03'),
('sample', 'location2', 'above baseline',
'ae168d5bfdad1463672120d51787fed2'),
('sample', 'location3', 'above baseline',
'380b79bea27bfa66e8864cfec9e3403a'),
('point', 'location1', 'Season peak week',
'fad04bc4cd443ca7cd7cd53f5de4fa99'),
('bin', 'location1', 'Season peak week',
'c74e3f626224eeb482368d9fb7a387da'),
('sample', 'location1', 'Season peak week',
'2f0fdc8a293046d38eb912601cf0a5cf'),
('point', 'location2', 'Season peak week',
'39c511635eb21cfde3657ab144521b94'),
('bin', 'location2', 'Season peak week',
'4fa62ed754c3fc9b9ede90926efe8f7f'),
('quantile', 'location2', 'Season peak week',
'd06cb30665b099e471c6dd9d50ba2c30'),
('point', 'location3', 'Season peak week',
'f15fab078daf9adb53f464272b31dbf6'),
('sample', 'location3', 'Season peak week',
'213d829834bceaaa4376a79b989161c3'),
]
pred_data_qs = PredictionElement.objects \
.filter(forecast=forecast) \
.values_list('pred_class', 'unit__name', 'target__name', 'data_hash') \
.order_by('id')
act_rows = [(PredictionElement.prediction_class_int_as_str(row[0]),
row[1], row[2], row[3]) for row in pred_data_qs]
self.assertEqual(sorted(exp_rows), sorted(act_rows))
def test_create_project_from_json_target_optional_fields(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
first_target_dict = project_dict['targets'][0] # 'Season onset'
project_dict['targets'] = [first_target_dict]
# test optional 'step_ahead_increment': required only if 'is_step_ahead'
first_target_dict[
'is_step_ahead'] = True # was False w/no 'step_ahead_increment'
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn(
"step_ahead_increment not found but is required when is_step_ahead",
str(context.exception))
# test optional fields, based on type:
# 1) test optional 'unit'. three cases a-c follow
# 1a) required but not passed: ['continuous', 'discrete', 'date']
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
first_target_dict = project_dict['targets'][0] # 'Season onset'
project_dict['targets'] = [first_target_dict]
for target_type in ['continuous', 'discrete', 'date']:
first_target_dict['type'] = target_type
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn(
f"'unit' not passed but is required for type_name={target_type}",
str(context.exception))
# 1b) optional: ok to pass or not pass: []: no need to validate
# 1c) invalid but passed: ['nominal', 'binary']
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
first_target_dict = project_dict['targets'][0] # 'Season onset'
project_dict['targets'] = [first_target_dict]
first_target_dict['unit'] = 'month'
for target_type in ['nominal', 'binary']:
first_target_dict['type'] = target_type
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn(
f"'unit' passed but is invalid for type_name={target_type}",
str(context.exception))
# 2) test optional 'range'. three cases a-c follow
# 2a) required but not passed: []: no need to validate
# 2b) optional: ok to pass or not pass: ['continuous', 'discrete']: no need to validate
# 2c) invalid but passed: ['nominal', 'binary', 'date']
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
first_target_dict = project_dict['targets'][0] # 'Season onset'
project_dict['targets'] = [first_target_dict]
first_target_dict['range'] = [1, 2]
for target_type in ['nominal', 'binary', 'date']:
first_target_dict['type'] = target_type
if target_type == 'date':
first_target_dict['unit'] = 'biweek'
else:
first_target_dict.pop('unit', None)
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn(
f"'range' passed but is invalid for type_name={target_type}",
str(context.exception))
# 3) test optional 'cats'. three cases a-c follow
# 3a) required but not passed: ['nominal', 'date']
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
first_target_dict = project_dict['targets'][0] # 'Season onset'
project_dict['targets'] = [first_target_dict]
first_target_dict.pop('cats', None)
first_target_dict.pop('unit', None)
for target_type in ['nominal', 'date']:
first_target_dict['type'] = target_type
if target_type in ['continuous', 'discrete', 'date']:
first_target_dict['unit'] = 'biweek'
else:
first_target_dict.pop('unit', None)
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn(
f"'cats' not passed but is required for type_name='{target_type}'",
str(context.exception))
# 3b) optional: ok to pass or not pass: ['continuous', 'discrete']: no need to validate
# 3c) invalid but passed: ['binary']
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
first_target_dict = project_dict['targets'][0] # 'Season onset'
project_dict['targets'] = [first_target_dict]
first_target_dict['cats'] = ['a', 'b']
for target_type in ['binary']:
first_target_dict['type'] = target_type
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn(
f"'cats' passed but is invalid for type_name={target_type}",
str(context.exception))
