def test_create_project_from_json_target_range_format(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
project_dict = json.load(fp)
pct_next_week_target_dict = [
target_dict for target_dict in project_dict['targets']
if target_dict['name'] == 'pct next week'
][0]
project_dict['targets'] = [pct_next_week_target_dict]
# loaded range is valid format: test that an error is not raised
try:
project = create_project_from_json(project_dict, po_user)
project.delete()
except Exception as ex:
self.fail(f"unexpected exception: {ex}")
# break range by setting to invalid format: test that an error is raised
range_list = ["not float", True] # not floats
pct_next_week_target_dict['range'] = range_list
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("range type did not match data_type",
str(context.exception))
# test exactly two items
range_list = [1.0, 2.2, 3.3] # 3, not 2
pct_next_week_target_dict['range'] = range_list
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("range did not contain exactly two items",
str(context.exception))
def test_target_round_trip_target_dict(self):
# test round trip: target_dict -> Target -> target_dict
# 1. target_dict -> Target
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
project_dict = json.load(fp)
input_target_dicts = project_dict['targets']
# does _validate_and_create_targets() -> model_init = {...} # required keys:
project = create_project_from_json(project_dict, po_user)
# 2. Target -> target_dict
# does target_dict() = {...} # required keys:
# note: using APIRequestFactory was the only way I could find to pass a request object. o/w you get:
# AssertionError: `HyperlinkedIdentityField` requires the request in the serializer context.
output_target_dicts = [
_target_dict_for_target(target,
APIRequestFactory().request())
for target in project.targets.all()
]
# 3. they should be equal
for target_dict in output_target_dicts: # remove 'id' and 'url' fields from TargetSerializer to ease testing
del target_dict['id']
del target_dict['url']
self.assertEqual(sorted(input_target_dicts, key=lambda _: _['name']),
sorted(output_target_dicts, key=lambda _: _['name']))
def test_last_update(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, time_zero, forecast_model, forecast = _make_docs_project(
po_user)
# one truth and one forecast (yes truth, yes forecasts)
self.assertEqual(forecast.created_at, project.last_update())
# add a second forecast for a newer timezero (yes truth, yes forecasts)
time_zero2 = TimeZero.objects.create(project=project,
timezero_date=datetime.date(
2011, 10, 3))
forecast2 = Forecast.objects.create(
forecast_model=forecast_model,
source='docs-predictions-non-dup.json',
time_zero=time_zero2,
notes="a small prediction file")
with open(
'forecast_app/tests/predictions/docs-predictions-non-dup.json'
) as fp:
json_io_dict_in = json.load(fp)
load_predictions_from_json_io_dict(forecast2,
json_io_dict_in,
is_validate_cats=False)
self.assertEqual(forecast2.created_at, project.last_update())
def test__upload_forecast_worker_blue_sky(self):
# blue sky to verify load_predictions_from_json_io_dict() and cache_forecast_metadata() are called. also tests
# that _upload_forecast_worker() correctly sets job.output_json. this test is complicated by that function's use
# of the `job_cloud_file` context manager. solution is per https://stackoverflow.com/questions/60198229/python-patch-context-manager-to-return-object
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, time_zero, forecast_model, forecast = _make_docs_project(
po_user)
forecast.issued_at -= datetime.timedelta(
days=1) # older version avoids unique constraint errors
forecast.save()
with patch('forecast_app.models.job.job_cloud_file') as job_cloud_file_mock, \
patch('utils.forecast.load_predictions_from_json_io_dict') as load_preds_mock, \
patch('utils.forecast.cache_forecast_metadata') as cache_metatdata_mock, \
open('forecast_app/tests/predictions/docs-predictions.json') as cloud_file_fp:
job = Job.objects.create()
job.input_json = {
'forecast_pk': forecast.pk,
'filename': 'a name!'
}
job.save()
job_cloud_file_mock.return_value.__enter__.return_value = (
job, cloud_file_fp)
_upload_forecast_worker(job.pk)
job.refresh_from_db()
load_preds_mock.assert_called_once()
cache_metatdata_mock.assert_called_once()
self.assertEqual(Job.SUCCESS, job.status)
self.assertEqual(job.input_json['forecast_pk'],
job.output_json['forecast_pk'])
def test_create_project_from_json_bad_arg(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
timezero_config = {
'timezero_date': '2017-12-01',
'data_version_date': None,
'is_season_start': True,
'season_name': 'tis the season'
}
project_dict['timezeros'] = [timezero_config]
# note: blue sky args (dict or Path) are checked elsewhere
bad_arg_exp_error = [
([1, 2],
'proj_config_file_path_or_dict was neither a dict nor a Path'),
('hi there',
'proj_config_file_path_or_dict was neither a dict nor a Path'),
(Path('forecast_app/tests/truth_data/truths-ok.csv'),
'error loading json file')
]
for bad_arg, exp_error in bad_arg_exp_error:
with self.assertRaises(RuntimeError) as context:
create_project_from_json(bad_arg, po_user)
self.assertIn(exp_error, str(context.exception))
def test_email_notification(self):
# test that Job.save() calls send_notification_email() for Job.FAILED, but not any others
# including Job.SUCCESS
_, _, mo_user, mo_user_password, _, _ = get_or_create_super_po_mo_users(
is_create_super=False)
self.client.login(username=mo_user.username, password=mo_user_password)
mo_user.email = "*****@*****.**"
with patch('forecast_app.notifications.send_notification_email'
) as send_email_mock:
# test Job.PENDING
Job.objects.create(user=mo_user, status=Job.PENDING)
send_email_mock.assert_not_called()
# test Job.SUCCESS
send_email_mock.reset_mock()
Job.objects.create(user=mo_user, status=Job.SUCCESS)
send_email_mock.assert_not_called()
# test Job.FAILED
send_email_mock.reset_mock()
job = Job.objects.create(user=mo_user, status=Job.FAILED)
address, subject, message = address_subject_message_for_job(job)
send_email_mock.assert_called_once_with(address, subject, message)
# test no user or no user email
mo_user.email = ''
mo_user.save()
self.assertIsNone(address_subject_message_for_job(job))
job = Job.objects.create(status=Job.FAILED) # no user
self.assertIsNone(address_subject_message_for_job(job))
def test_duplicate_abbreviation(self):
# duplicate names are OK, but not abbreviations
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, time_zero, forecast_model, forecast = _make_docs_project(
po_user)
# case: new name, duplicate abbreviation
with self.assertRaises(ValidationError) as context:
ForecastModel.objects.create(
project=project,
name=forecast_model.name + '2',
abbreviation=forecast_model.abbreviation)
self.assertIn('abbreviation must be unique', str(context.exception))
# case: duplicate name, new abbreviation
try:
forecast_model2 = ForecastModel.objects.create(
project=project,
name=forecast_model.name,
abbreviation=forecast_model.abbreviation + '2')
forecast_model2.delete()
except Exception as ex:
self.fail(f"unexpected exception: {ex}")
# case: new name, duplicate abbreviation, but saving existing forecast_model
try:
forecast_model.name = forecast_model.name + '2' # new name, duplicate abbreviation
forecast_model.save()
forecast_model.abbreviation = forecast_model.abbreviation + '2' # duplicate name, new abbreviation
forecast_model.save()
except Exception as ex:
self.fail(f"unexpected exception: {ex}")
def test_load_truth_data_versions(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, time_zero, forecast_model, forecast = _make_docs_project(
po_user) # loads docs-ground-truth.csv
oracle_model = oracle_model_for_project(project)
self.assertEqual(3, oracle_model.forecasts.count(
)) # for 3 timezeros: 2011-10-02, 2011-10-09, 2011-10-16
self.assertEqual(14, truth_data_qs(project).count())
self.assertTrue(is_truth_data_loaded(project))
with self.assertRaisesRegex(RuntimeError,
'cannot load 100% duplicate data'):
load_truth_data(
project,
Path('forecast_app/tests/truth_data/docs-ground-truth.csv'),
file_name='docs-ground-truth.csv')
load_truth_data(
project,
Path(
'forecast_app/tests/truth_data/docs-ground-truth-non-dup.csv'),
file_name='docs-ground-truth-non-dup.csv')
self.assertEqual(3 * 2, oracle_model.forecasts.count())
self.assertEqual(14 * 2, truth_data_qs(project).count())
def test_create_project_from_json_target_cats_format(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
project_dict = json.load(fp)
pct_next_week_target_dict = [
target_dict for target_dict in project_dict['targets']
if target_dict['name'] == 'pct next week'
][0]
project_dict['targets'] = [pct_next_week_target_dict]
# loaded cats is valid format: test that an error is not raised
try:
project = create_project_from_json(project_dict, po_user)
project.delete()
except Exception as ex:
self.fail(f"unexpected exception: {ex}")
# break cats by setting to invalid format: test that an error is raised
cats = ["not float", True, {}] # not floats
pct_next_week_target_dict['cats'] = cats
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn(
"could not convert cat to data_type. cat_str='not float'",
str(context.exception))
def test_load_predictions_from_json_io_dict(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(is_create_super=True)
project = create_project_from_json(Path('forecast_app/tests/projects/docs-project.json'), po_user)
forecast_model = ForecastModel.objects.create(project=project, name='name', abbreviation='abbrev')
time_zero = TimeZero.objects.create(project=project, timezero_date=datetime.date(2017, 1, 1))
forecast = Forecast.objects.create(forecast_model=forecast_model, source='docs-predictions.json',
time_zero=time_zero)
# test json with no 'predictions'
with self.assertRaises(RuntimeError) as context:
load_predictions_from_json_io_dict(forecast, {}, False)
self.assertIn("json_io_dict had no 'predictions' key", str(context.exception))
# load all four types of Predictions, call Forecast.*_qs() functions. see docs-predictionsexp-rows.xlsx.
# counts from docs-predictionsexp-rows.xlsx: point: 11, named: 3, bin: 30 (3 zero prob), sample: 23
# = total rows: 67
#
# counts based on .json file:
# - 'pct next week': point: 3, named: 1 , bin: 3, sample: 5, quantile: 5 = 17
# - 'cases next week': point: 2, named: 1 , bin: 3, sample: 3, quantile: 2 = 12
# - 'season severity': point: 2, named: 0 , bin: 3, sample: 5, quantile: 0 = 10
# - 'above baseline': point: 1, named: 0 , bin: 2, sample: 6, quantile: 0 = 9
# - 'Season peak week': point: 3, named: 0 , bin: 7, sample: 4, quantile: 3 = 16
# = total rows: 64 - 2 zero prob = 62
with open('forecast_app/tests/predictions/docs-predictions.json') as fp:
json_io_dict = json.load(fp)
load_predictions_from_json_io_dict(forecast, json_io_dict, False)
self.assertEqual(62, forecast.get_num_rows())
self.assertEqual(16, forecast.bin_distribution_qs().count()) # 18 - 2 zero prob
self.assertEqual(2, forecast.named_distribution_qs().count())
self.assertEqual(11, forecast.point_prediction_qs().count())
self.assertEqual(23, forecast.sample_distribution_qs().count())
self.assertEqual(10, forecast.quantile_prediction_qs().count())
def test_prediction_dicts_to_db_rows_invalid(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(is_create_super=True)
project = create_project_from_json(Path('forecast_app/tests/projects/docs-project.json'), po_user)
forecast_model = ForecastModel.objects.create(project=project, name='name', abbreviation='abbrev')
time_zero = TimeZero.objects.create(project=project, timezero_date=datetime.date(2017, 1, 1))
forecast = Forecast.objects.create(forecast_model=forecast_model, source='docs-predictions.json',
time_zero=time_zero)
# test for invalid unit
with self.assertRaises(RuntimeError) as context:
bad_prediction_dicts = [
{"unit": "bad unit", "target": "1 wk ahead", "class": "BinCat", "prediction": {}}
]
_prediction_dicts_to_validated_db_rows(forecast, bad_prediction_dicts, False)
self.assertIn('prediction_dict referred to an undefined Unit', str(context.exception))
# test for invalid target
with self.assertRaises(RuntimeError) as context:
bad_prediction_dicts = [
{"unit": "location1", "target": "bad target", "class": "bad class", "prediction": {}}
]
_prediction_dicts_to_validated_db_rows(forecast, bad_prediction_dicts, False)
self.assertIn('prediction_dict referred to an undefined Target', str(context.exception))
# test for invalid prediction_class
with self.assertRaises(RuntimeError) as context:
bad_prediction_dicts = [
{"unit": "location1", "target": "pct next week", "class": "bad class", "prediction": {}}
]
_prediction_dicts_to_validated_db_rows(forecast, bad_prediction_dicts, False)
self.assertIn('invalid prediction_class', str(context.exception))
def test_diff_from_file(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, _, _, _ = _make_docs_project(po_user)
out_config_dict = config_dict_from_project(
project,
APIRequestFactory().request())
# this json file makes the same changes as _make_some_changes():
with open(
Path('forecast_app/tests/project_diff/docs-project-edited.json'
)) as fp:
edited_config_dict = json.load(fp)
changes = project_config_diff(out_config_dict, edited_config_dict)
# # print a little report
# print(f"* Analyzed {len(changes)} changes. Results:")
# for change, num_points, num_named, num_bins, num_samples, num_quantiles, num_truth in \
# database_changes_for_project_config_diff(project, changes):
# print(f"- {change.change_type.name} on {change.object_type.name} {change.object_pk!r} will delete:\n"
# f" = {num_points} point predictions\n"
# f" = {num_named} named predictions\n"
# f" = {num_bins} bin predictions\n"
# f" = {num_samples} samples\n"
# f" = {num_quantiles} quantiles\n"
# f" = {num_truth} truth rows")
# same tests as test_execute_project_config_diff():
execute_project_config_diff(project, changes)
self._do_make_some_changes_tests(project)
def test_target_range_cat_validation(self):
# tests this relationship: "If `cats` are specified, then the min(`cats`) must equal the lower bound of `range`
# and max(`cats`) must be less than the upper bound of `range`."
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
input_project_dict = json.load(fp)
# test: "the min(`cats`) must equal the lower bound of `range`":
# for the "cases next week" target, change min(cats) to != min(range)
# "range": [0, 100000]
# "cats": [0, 2, 50] -> change to [1, 2, 50]
input_project_dict['targets'][1]['cats'] = [1, 2, 50]
with self.assertRaises(RuntimeError) as context:
create_project_from_json(input_project_dict, po_user)
self.assertIn("the minimum cat (1) did not equal the range's lower bound (0)", str(context.exception))
# test: "max(`cats`) must be less than the upper bound of `range`":
# for the "cases next week" target, change max(cats) to == max(range)
# "range": [0, 100000]
# "cats": [0, 2, 50] -> change to [0, 2, 100000]
input_project_dict['targets'][1]['cats'] = [0, 2, 100000]
with self.assertRaises(RuntimeError) as context:
create_project_from_json(input_project_dict, po_user)
self.assertIn("the maximum cat (100000) was not less than the range's upper bound", str(context.exception))
# also test max(cats) to > max(range)
input_project_dict['targets'][1]['cats'] = [0, 2, 100001]
with self.assertRaises(RuntimeError) as context:
create_project_from_json(input_project_dict, po_user)
self.assertIn("the maximum cat (100001) was not less than the range's upper bound ", str(context.exception))
def setUpTestData(cls):
# recall that _make_docs_project() calls cache_forecast_metadata(), but the below tests assume it doesn't, so
# we clear here
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(is_create_super=True)
cls.project, cls.time_zero, cls.forecast_model, cls.forecast = _make_docs_project(po_user)
clear_forecast_metadata(cls.forecast)
cls.forecast.issued_at -= datetime.timedelta(days=1) # older version avoids unique constraint errors
cls.forecast.save()
def test_load_predictions_from_json_io_dict(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), po_user)
forecast_model = ForecastModel.objects.create(project=project,
name='name',
abbreviation='abbrev')
time_zero = TimeZero.objects.create(project=project,
timezero_date=datetime.date(
2017, 1, 1))
forecast = Forecast.objects.create(forecast_model=forecast_model,
source='docs-predictions.json',
time_zero=time_zero)
# test json with no 'predictions'
with self.assertRaises(RuntimeError) as context:
load_predictions_from_json_io_dict(forecast, {},
is_validate_cats=False)
self.assertIn("json_io_dict had no 'predictions' key",
str(context.exception))
# test loading all five types of Predictions
with open(
'forecast_app/tests/predictions/docs-predictions.json') as fp:
json_io_dict = json.load(fp)
load_predictions_from_json_io_dict(forecast,
json_io_dict,
is_validate_cats=False)
# test prediction element counts match number in .json file
pred_ele_qs = forecast.pred_eles.all()
pred_data_qs = PredictionData.objects.filter(
pred_ele__forecast=forecast)
self.assertEqual(29, len(pred_ele_qs))
self.assertEqual(29, len(pred_data_qs))
# test there's a prediction element for every .json item
unit_name_to_obj = {unit.name: unit for unit in project.units.all()}
target_name_to_obj = {
target.name: target
for target in project.targets.all()
}
for pred_ele_dict in json_io_dict['predictions']:
unit = unit_name_to_obj[pred_ele_dict['unit']]
target = target_name_to_obj[pred_ele_dict['target']]
pred_class_int = PRED_CLASS_NAME_TO_INT[pred_ele_dict['class']]
data_hash = PredictionElement.hash_for_prediction_data_dict(
pred_ele_dict['prediction'])
pred_ele = pred_ele_qs.filter(pred_class=pred_class_int,
unit=unit,
target=target,
is_retract=False,
data_hash=data_hash).first()
self.assertIsNotNone(pred_ele)
self.assertIsNotNone(
pred_data_qs.filter(pred_ele=pred_ele).first())
def test_models_summary_table_rows_for_project(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, time_zero, forecast_model, forecast = _make_docs_project(
po_user)
# test with just one forecast - oldest and newest forecast is the same. a 7-tuple:
# [forecast_model, num_forecasts, oldest_forecast_tz_date, newest_forecast_tz_date, oldest_forecast_id,
# newest_forecast_id, newest_forecast_created_at].
# NB: we have to work around a Django bug where DateField and DateTimeField come out of the database as either
# datetime.date/datetime.datetime objects (postgres) or strings (sqlite3)
exp_row = (
forecast_model,
forecast_model.forecasts.count(),
str(time_zero.timezero_date), # oldest_forecast_tz_date
str(time_zero.timezero_date), # newest_forecast_tz_date
forecast.id,
forecast.created_at.utctimetuple()) # newest_forecast_created_at
act_rows = models_summary_table_rows_for_project(project)
act_rows = [(
act_rows[0][0],
act_rows[0][1],
str(act_rows[0][2]), # oldest_forecast_tz_date
str(act_rows[0][3]), # newest_forecast_tz_date
act_rows[0][4],
act_rows[0][5].utctimetuple())] # newest_forecast_created_at
sql = f"""SELECT created_at FROM {Forecast._meta.db_table} WHERE id = %s;"""
with connection.cursor() as cursor:
cursor.execute(sql, (forecast.pk, ))
rows = cursor.fetchall()
self.assertEqual([exp_row], act_rows)
# test a second forecast
time_zero2 = TimeZero.objects.create(project=project,
timezero_date=datetime.date(
2017, 1, 1))
forecast2 = Forecast.objects.create(forecast_model=forecast_model,
source='docs-predictions.json',
time_zero=time_zero2)
exp_row = (
forecast_model,
forecast_model.forecasts.count(),
str(time_zero.timezero_date), # oldest_forecast_tz_date
str(time_zero2.timezero_date), # newest_forecast_tz_date
forecast2.id,
forecast2.created_at.utctimetuple()) # newest_forecast_created_at
act_rows = models_summary_table_rows_for_project(project)
act_rows = [(
act_rows[0][0],
act_rows[0][1],
str(act_rows[0][2]), # oldest_forecast_tz_date
str(act_rows[0][3]), # newest_forecast_tz_date
act_rows[0][4],
act_rows[0][5].utctimetuple())] # newest_forecast_created_at
self.assertEqual([exp_row], act_rows)
def test_truth_batches(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, time_zero, forecast_model, forecast = _make_docs_project(
po_user) # loads batch: docs-ground-truth.csv
# add a second batch
load_truth_data(
project,
Path(
'forecast_app/tests/truth_data/docs-ground-truth-non-dup.csv'),
file_name='docs-ground-truth-non-dup.csv')
oracle_model = oracle_model_for_project(project)
first_forecast = oracle_model.forecasts.first()
last_forecast = oracle_model.forecasts.last()
# test truth_batches() and truth_batch_forecasts() for each batch
batches = truth_batches(project)
self.assertEqual(2, len(batches))
self.assertEqual(first_forecast.source, batches[0][0])
self.assertEqual(first_forecast.issued_at, batches[0][1])
self.assertEqual(last_forecast.source, batches[1][0])
self.assertEqual(last_forecast.issued_at, batches[1][1])
for source, issued_at in batches:
forecasts = truth_batch_forecasts(project, source, issued_at)
self.assertEqual(3, len(forecasts))
for forecast in forecasts:
self.assertEqual(source, forecast.source)
self.assertEqual(issued_at, forecast.issued_at)
# test truth_batch_summary_table(). NB: utctimetuple() makes sqlite comparisons work
exp_table = [(source, issued_at.utctimetuple(),
len(truth_batch_forecasts(project, source, issued_at)))
for source, issued_at in batches]
act_table = [(source, issued_at.utctimetuple(), num_forecasts)
for source, issued_at, num_forecasts in
truth_batch_summary_table(project)]
self.assertEqual(exp_table, act_table)
# finally, test deleting a batch. try deleting the first, which should fail due to version rules.
# transaction.atomic() somehow avoids the second `truth_delete_batch()` call getting the error:
# django.db.transaction.TransactionManagementError: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.
with transaction.atomic():
with self.assertRaisesRegex(
RuntimeError,
'you cannot delete a forecast that has any newer versions'
):
truth_delete_batch(project, batches[0][0], batches[0][1])
# delete second batch - should not fail
truth_delete_batch(project, batches[1][0], batches[1][1])
batches = truth_batches(project)
self.assertEqual(1, len(batches))
self.assertEqual(first_forecast.source, batches[0][0])
self.assertEqual(first_forecast.issued_at, batches[0][1])
def test_create_project_from_json_duplicate_target(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
project_dict = json.load(fp)
project_dict['targets'].append(project_dict['targets'][0])
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("found existing Target for name", str(context.exception))
def test_create_project_from_json(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
timezero_config = {
'timezero_date': '2017-12-01',
'data_version_date': None,
'is_season_start': True,
'season_name': 'tis the season'
}
project_dict['timezeros'] = [timezero_config]
project = create_project_from_json(project_dict, po_user)
# spot-check some fields
self.assertEqual(po_user, project.owner)
self.assertTrue(project.is_public)
self.assertEqual('CDC Flu challenge', project.name)
self.assertEqual(Project.WEEK_TIME_INTERVAL_TYPE,
project.time_interval_type)
self.assertEqual('Weighted ILI (%)', project.visualization_y_label)
self.assertEqual(11, project.units.count())
self.assertEqual(7, project.targets.count())
# spot-check a Unit
unit = project.units.filter(name='US National').first()
self.assertIsNotNone(unit)
# spot-check a Target
target = project.targets.filter(name='1 wk ahead').first()
self.assertEqual(Target.CONTINUOUS_TARGET_TYPE, target.type)
self.assertEqual('percent', target.unit)
self.assertTrue(target.is_step_ahead)
self.assertEqual(1, target.step_ahead_increment)
# check the TimeZero
time_zero = project.timezeros.first()
self.assertIsNotNone(time_zero)
self.assertEqual(datetime.date(2017, 12, 1), time_zero.timezero_date)
self.assertIsNone(time_zero.data_version_date)
self.assertEqual(timezero_config['is_season_start'],
time_zero.is_season_start)
self.assertEqual(timezero_config['season_name'], time_zero.season_name)
# test existing project
project.delete()
with open('forecast_app/tests/projects/cdc-project.json') as fp:
cdc_project_json = json.load(fp)
create_project_from_json(
Path('forecast_app/tests/projects/cdc-project.json'), po_user)
with self.assertRaises(RuntimeError) as context:
create_project_from_json(cdc_project_json, po_user)
self.assertIn("found existing project", str(context.exception))
def test_range_tuple(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(is_create_super=True)
with open(Path('forecast_app/tests/projects/docs-project.json')) as fp:
input_project_dict = json.load(fp)
project = create_project_from_json(input_project_dict, po_user)
act_range_tuples = [(target.name, target.range_tuple()) for target in project.targets.all().order_by('pk')]
self.assertEqual([('pct next week', (0.0, 100.0)),
('cases next week', (0, 100000)),
('season severity', None),
('above baseline', None),
('Season peak week', None)],
act_range_tuples)
def setUp(
self
): # runs before every test. done here instead of setUpTestData(cls) b/c below tests modify the db
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
self.project, self.time_zero, self.forecast_model, self.forecast = _make_docs_project(
po_user)
self.tz1 = self.project.timezeros.filter(
timezero_date=datetime.date(2011, 10, 2)).first()
self.tz2 = self.project.timezeros.filter(
timezero_date=datetime.date(2011, 10, 9)).first()
self.tz3 = self.project.timezeros.filter(
timezero_date=datetime.date(2011, 10, 16)).first()
def test_execute_project_config_diff(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, _, _, _ = _make_docs_project(po_user)
# make some changes
out_config_dict = config_dict_from_project(
project,
APIRequestFactory().request())
edit_config_dict = copy.deepcopy(out_config_dict)
_make_some_changes(edit_config_dict)
changes = project_config_diff(out_config_dict, edit_config_dict)
execute_project_config_diff(project, changes)
self._do_make_some_changes_tests(project)
def test_query_truth_for_project_null_rows(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), po_user)
load_truth_data(
project,
Path(
'forecast_app/tests/truth_data/docs-ground-truth-null-value.csv'
),
is_convert_na_none=True)
exp_rows = [-1] # todo xx
act_rows = list(query_truth_for_project(project, {}))
self.assertEqual(sorted(exp_rows), sorted(act_rows))
def test__upload_forecast_worker_atomic(self):
# test `_upload_forecast_worker()` does not create a Forecast if subsequent calls to
# `load_predictions_from_json_io_dict()` or `cache_forecast_metadata()` fail. this test is complicated by that
# function's use of the `job_cloud_file` context manager. solution is per https://stackoverflow.com/questions/60198229/python-patch-context-manager-to-return-object
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, time_zero, forecast_model, forecast = _make_docs_project(
po_user)
forecast.issued_at -= datetime.timedelta(
days=1) # older version avoids unique constraint errors
forecast.save()
with patch('forecast_app.models.job.job_cloud_file') as job_cloud_file_mock, \
patch('utils.forecast.load_predictions_from_json_io_dict') as load_preds_mock, \
patch('utils.forecast.cache_forecast_metadata') as cache_metatdata_mock:
forecast2 = Forecast.objects.create(forecast_model=forecast_model,
time_zero=time_zero)
job = Job.objects.create()
job.input_json = {
'forecast_pk': forecast2.pk,
'filename': 'a name!'
}
job.save()
job_cloud_file_mock.return_value.__enter__.return_value = (
job, None) # 2-tuple: (job, cloud_file_fp)
# test that no Forecast is created when load_predictions_from_json_io_dict() fails
load_preds_mock.side_effect = Exception(
'load_preds_mock Exception')
num_forecasts_before = forecast_model.forecasts.count()
_upload_forecast_worker(job.pk)
job.refresh_from_db()
self.assertEqual(
num_forecasts_before - 1,
forecast_model.forecasts.count()) # -1 b/c forecast2 deleted
self.assertEqual(Job.FAILED, job.status)
# test when cache_forecast_metadata() fails
load_preds_mock.reset_mock(side_effect=True)
cache_metatdata_mock.side_effect = Exception(
'cache_metatdata_mock Exception')
num_forecasts_before = forecast_model.forecasts.count()
_upload_forecast_worker(job.pk)
job.refresh_from_db()
self.assertEqual(num_forecasts_before,
forecast_model.forecasts.count())
self.assertEqual(Job.FAILED, job.status)
def test_create_project_from_json_target_types(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
# test valid types
minimal_target_dict = {
'name': 'n',
'description': 'd',
'is_step_ahead': False
} # no 'type'
target_type_int_to_required_keys_and_values = {
Target.CONTINUOUS_TARGET_TYPE:
[('unit', 'month')], # 'range' optional
Target.DISCRETE_TARGET_TYPE:
[('unit', 'month')], # 'range' optional
Target.NOMINAL_TARGET_TYPE: [('cats', ['a', 'b'])],
Target.BINARY_TARGET_TYPE: [], # binary has no required keys
Target.DATE_TARGET_TYPE: [('unit', 'month'),
('cats', ['2019-12-15', '2019-12-22'])]
}
type_int_to_name = {
type_int: type_name
for type_int, type_name in Target.TARGET_TYPE_CHOICES
}
for type_int, required_keys_and_values in target_type_int_to_required_keys_and_values.items(
):
test_target_dict = dict(minimal_target_dict) # shallow copy
project_dict['targets'] = [test_target_dict]
test_target_dict['type'] = type_int_to_name[type_int]
for required_key, value in required_keys_and_values:
test_target_dict[required_key] = value
project = create_project_from_json(project_dict, po_user)
project.delete()
# test invalid type
Project.objects.filter(name=project_dict['name']).delete()
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
first_target_dict = project_dict['targets'][0] # 'Season onset'
project_dict['targets'] = [first_target_dict]
first_target_dict['type'] = 'invalid type'
with self.assertRaises(RuntimeError) as context:
create_project_from_json(project_dict, po_user)
self.assertIn("Invalid type_name", str(context.exception))
def test_load_truth_data_null_rows(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project = create_project_from_json(
Path('forecast_app/tests/projects/docs-project.json'), po_user)
load_truth_data(
project,
Path(
'forecast_app/tests/truth_data/docs-ground-truth-null-value.csv'
),
is_convert_na_none=True)
exp_rows = [
(datetime.date(2011, 10, 2), 'location1', 'Season peak week', None,
None, None, datetime.date(2019, 12, 15), None),
(datetime.date(2011, 10, 2), 'location1', 'above baseline', None,
None, None, None, True),
(datetime.date(2011, 10, 2), 'location1', 'season severity', None,
None, 'moderate', None, None),
(datetime.date(2011, 10, 2), 'location1', 'cases next week', None,
None, None, None, None), # all None
(datetime.date(2011, 10, 2), 'location1', 'pct next week', None,
None, None, None, None), # all None
(datetime.date(2011, 10, 9), 'location2', 'Season peak week', None,
None, None, datetime.date(2019, 12, 29), None),
(datetime.date(2011, 10, 9), 'location2', 'above baseline', None,
None, None, None, True),
(datetime.date(2011, 10, 9), 'location2', 'season severity', None,
None, 'severe', None, None),
(datetime.date(2011, 10, 9), 'location2', 'cases next week', 3,
None, None, None, None),
(datetime.date(2011, 10, 9), 'location2', 'pct next week', None,
99.9, None, None, None),
(datetime.date(2011, 10, 16), 'location1', 'Season peak week',
None, None, None, datetime.date(2019, 12, 22), None),
(datetime.date(2011, 10, 16), 'location1', 'above baseline', None,
None, None, None, False),
(datetime.date(2011, 10, 16), 'location1', 'cases next week', 0,
None, None, None, None),
(datetime.date(2011, 10, 16), 'location1', 'pct next week', None,
0.0, None, None, None)
]
act_rows = truth_data_qs(project) \
.values_list('pred_ele__forecast__time_zero__timezero_date',
'pred_ele__unit__name', 'pred_ele__target__name',
'value_i', 'value_f', 'value_t', 'value_d', 'value_b')
self.assertEqual(sorted(exp_rows), sorted(act_rows))
def test_load_truth_data_partial_dup(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, time_zero, forecast_model, forecast = _make_docs_project(
po_user) # loads batch: docs-ground-truth.csv
try:
load_truth_data(
project,
Path(
'forecast_app/tests/truth_data/docs-ground-truth-partial-dup.csv'
),
file_name='docs-ground-truth-partial-dup.csv')
batches = truth_batches(project)
self.assertEqual(2, len(batches))
except Exception as ex:
self.fail(f"unexpected exception: {ex}")
def test_order_project_config_diff(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, _, _, _ = _make_docs_project(po_user)
out_config_dict = config_dict_from_project(
project,
APIRequestFactory().request())
edit_config_dict = copy.deepcopy(out_config_dict)
_make_some_changes(edit_config_dict)
changes = project_config_diff(out_config_dict, edit_config_dict)
# removes one wasted activity ('pct next week', ChangeType.FIELD_EDITED) that is wasted b/c that target is being
# ChangeType.OBJ_REMOVED:
ordered_changes = order_project_config_diff(changes)
self.assertEqual(
13, len(changes)) # contains two duplicate and one wasted change
self.assertEqual(10, len(ordered_changes))
def test_diff_from_file_empty_data_version_date_string(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
project, _, _, _ = _make_docs_project(po_user)
# note: using APIRequestFactory was the only way I could find to pass a request object. o/w you get:
# AssertionError: `HyperlinkedIdentityField` requires the request in the serializer context.
out_config_dict = config_dict_from_project(
project,
APIRequestFactory().request())
edited_config_dict = copy.deepcopy(out_config_dict)
# change '2011-10-02': None -> '' (incorrect, but we fix for users)
edited_config_dict['timezeros'][0]['data_version_date'] = ''
changes = project_config_diff(out_config_dict, edited_config_dict)
self.assertEqual(
0, len(changes)
) # is 1 without the fix "this test for `!= ''` matches this one below"
def test_create_project_from_json_target_required_fields(self):
_, _, po_user, _, _, _, _, _ = get_or_create_super_po_mo_users(
is_create_super=True)
with open(Path('forecast_app/tests/projects/cdc-project.json')) as fp:
project_dict = json.load(fp)
first_target_dict = project_dict['targets'][0] # 'Season onset'
project_dict['targets'] = [first_target_dict]
# test missing target required fields. optional keys are tested below
for field_name in ['name', 'description', 'type',
'is_step_ahead']: # required
field_value = first_target_dict[field_name]
with self.assertRaises(RuntimeError) as context:
del (first_target_dict[field_name])
create_project_from_json(project_dict, po_user)
self.assertIn("Wrong required keys in target_dict",
str(context.exception))
first_target_dict[field_name] = field_value # reset to valid
