def test_run_calls_update(self):
stk = hotspot.SKernelHistoric(1, bdwidth=0.3)
# need to train model before running, otherwise it won't get past the call to the mocked function
vb = validation.ValidationBase(self.data, stk)
vb.set_sample_units(0.1)
vb.train_model()
# check correct calls being made to _update
with mock.patch.object(validation.ValidationBase, '_update') as m:
t0 = vb.cutoff_t
res = vb.run(time_step=0.1)
# _update is the time updater, so cutoff time should not have changed
self.assertEqual(vb.cutoff_t, t0)
expct_call_count = math.ceil((1 - t0) / 0.1)
self.assertEqual(m.call_count, expct_call_count)
self.assertEqual(
m.call_args_list[0],
mock.call(time_step=0.0)) # _initial_setup routes to here
self.assertEqual(m.call_args, mock.call(0.1)) # normal operation
stk = hotspot.SKernelHistoric(1, bdwidth=0.3)
vb = validation.ValidationBase(self.data, stk)
expct_call_count = math.ceil((1 - vb.cutoff_t) / 0.1)
def test_instantiation(self):
# shuffle data to check that it is resorted
data = np.array(self.data)
np.random.shuffle(data)
stk = hotspot.SKernelHistoric(1, bdwidth=0.3)
vb = validation.ValidationBase(data, stk)
# check data are present and sorted
self.assertEqual(vb.ndata, self.data.shape[0])
self.assertTrue(np.all(np.diff(vb.data[:, 0]) > 0))
self.assertTrue(np.all(vb.data == self.data))
# no grid present yet as not supplied
self.assertTrue(vb.roc.poly is None)
# instantiate grid
vb.set_sample_units(0.1)
# should now have automatically made a spatial domain
self.assertListEqual(list(vb.roc.poly.bounds), [
min(self.data[:, 1]),
min(self.data[:, 2]),
max(self.data[:, 1]),
max(self.data[:, 2]),
])
# repeat but this time copy the grid
vb2 = validation.ValidationBase(self.data, stk)
self.assertTrue(vb2.roc.poly is None)
vb2.set_sample_units(vb.roc)
self.assertListEqual(vb.roc.sample_units, vb2.roc.sample_units)
def test_training(self):
stk = mock.create_autospec(hotspot.STKernelBowers)
vb = validation.ValidationBase(self.data, stk)
# check that model is NOT trained initially
self.assertEqual(stk.train.call_count, 0)
# spoof train
vb.train_model()
self.assertEqual(stk.train.call_count, 1)
self.assertEqual(len(stk.train.call_args[0]), 1)
self.assertTrue(np.all(stk.train.call_args[0][0] == vb.training))
vb.train_model()
self.assertEqual(stk.train.call_count, 2)
# set cutoff time, no training
vb.set_t_cutoff(0.1, b_train=False)
self.assertEqual(stk.train.call_count, 2)
# set cutoff time, training
vb.set_t_cutoff(0.1, b_train=True)
self.assertEqual(stk.train.call_count, 3)
self.assertEqual(len(stk.train.call_args[0]), 1)
self.assertTrue(np.all(stk.train.call_args[0][0] == vb.training))
def test_run_no_grid(self):
stk = hotspot.SKernelHistoric(1, bdwidth=0.3)
vb = validation.ValidationBase(self.data, stk)
t0 = vb.cutoff_t
# no grid specified raises error
with self.assertRaises(AssertionError):
res = vb.run(time_step=0.1)
vb.set_sample_units(0.1)
res = vb.run(time_step=0.1)
expct_call_count = math.ceil((1 - t0) / 0.1)
for k, v in res.items():
# only test the length of iterable values, as the res dict also contains some parameter values
# that are float or int
if hasattr(v, '__iter__'):
self.assertEqual(len(v), expct_call_count)
def test_predict(self):
stk = hotspot.SKernelHistoric(1, bdwidth=0.3)
vb = validation.ValidationBase(self.data, stk)
vb.train_model()
vb.set_sample_units(0.1)
# spoof predict at all grid centroids
stk = mock.create_autospec(hotspot.STKernelBowers)
vb.model = stk
pop = vb.predict(vb.cutoff_t)
self.assertEqual(stk.predict.call_count, 1)
sp = vb.sample_points
pred_call_arg = (vb.cutoff_t, vb.sample_points)
self.assertTrue(stk.predict.call_args[0] == pred_call_arg)
def test_assess(self):
stk = hotspot.SKernelHistoric(1, bdwidth=0.3)
vb = validation.ValidationBase(self.data, stk)
vb.train_model()
# mock roc object with grid
mocroc = mock.create_autospec(roc.RocGrid)
mocroc.centroids = np.array([[0., 0.], [1., 1.]])
mocroc.sample_units = range(
2) # needs to have the correct length, contents not used
mocroc.sample_points = DataArray([[0., 0.], [1., 1.]])
vb.roc = mocroc
res = vb._iterate_run(pred_dt_plus=0.2,
true_dt_plus=None,
true_dt_minus=0.)
# set data
self.assertTrue(vb.roc.set_data.called)
self.assertEqual(vb.roc.set_data.call_count, 1)
self.assertTrue(
np.all(vb.roc.set_data.call_args[0] == vb.testing(
dt_plus=0.2)[:, 1:]))
# set prediction
self.assertTrue(vb.roc.set_prediction.called)
self.assertEqual(vb.roc.set_prediction.call_count, 1)
self.assertEqual(len(vb.roc.set_prediction.call_args[0]), 1)
t = (vb.cutoff_t + 0.2)
# expected prediction values at (0,0), (1,1)
pred_arg = DataArray(np.array([[t, 0., 0.], [t, 1., 1.]]))
pred_expctd = vb.model.predict(t, mocroc.sample_points)
self.assertTrue(
np.all(vb.roc.set_prediction.call_args[0][0] == pred_expctd))
# set grid
self.assertFalse(vb.roc.set_sample_units.called)
vb.set_sample_units(0.1)
self.assertTrue(vb.roc.set_sample_units.called)
self.assertEqual(vb.roc.set_sample_units.call_count, 1)
self.assertTupleEqual(vb.roc.set_sample_units.call_args[0], (0.1, ))
# evaluate
self.assertTrue(vb.roc.evaluate.called)
self.assertEqual(vb.roc.evaluate.call_count, 1)
def test_run_calls_iterate_run(self):
stk = hotspot.SKernelHistoric(1, bdwidth=0.3)
# check correct calls are made to _iterate_run with no t_upper
with mock.patch.object(
validation.ValidationBase,
'_iterate_run',
return_value=collections.defaultdict(list)) as m:
vb = validation.ValidationBase(self.data, stk)
vb.set_sample_units(0.1)
t0 = vb.cutoff_t
vb.run(time_step=0.1)
expct_call_count = math.ceil((1 - t0) / 0.1)
self.assertEqual(m.call_count, expct_call_count)
for c in m.call_args_list:
self.assertEqual(
c, mock.call(0.1, 0.1, 0.)
) # signature: pred_dt_plus, true_dt_plus, true_dt_minus
def test_run_calls_initial_setup(self):
stk = hotspot.SKernelHistoric(1, bdwidth=0.3)
# need to train model before running, otherwise it won't get past the call to the mocked function
vb = validation.ValidationBase(self.data, stk)
vb.set_sample_units(0.1)
vb.train_model()
# check correct calls being made to _initial_setup
with mock.patch.object(validation.ValidationBase,
'_initial_setup') as m:
t0 = vb.cutoff_t
res = vb.run(time_step=0.1)
self.assertEqual(m.call_count, 1) # initial setup only called once
self.assertEqual(m.call_args, mock.call()) # no arguments passed
# test passing training keywords
train_kwargs = {'foo': 'bar'}
res = vb.run(time_step=0.1, train_kwargs=train_kwargs)
self.assertEqual(m.call_args, mock.call(**train_kwargs))
def test_time_cutoff(self):
stk = hotspot.SKernelHistoric(1, bdwidth=0.3)
vb = validation.ValidationBase(self.data, stk)
# expected cutoff automatically chosen
cutoff_te = vb.data[int(self.data.shape[0] / 2), 0]
# check time cutoff and training/test split
self.assertEqual(vb.cutoff_t, cutoff_te)
# training set
# training_expctd = self.data[self.data[:, 0] <= cutoff_te]
training_expctd = self.data[self.data[:, 0] < cutoff_te]
self.assertEqual(vb.training.ndata, len(training_expctd))
self.assertTrue(np.all(vb.training == training_expctd))
# testing set
# testing_expctd = self.data[self.data[:, 0] > cutoff_te]
testing_expctd = self.data[self.data[:, 0] >= cutoff_te]
self.assertEqual(vb.testing().ndata, len(testing_expctd))
self.assertTrue(np.all(vb.testing() == testing_expctd))
# test when as_point=True
tst = vb.testing(as_point=True)
self.assertEqual(len(tst), len(testing_expctd))
for i in range(len(testing_expctd)):
self.assertEqual(tst[i][0], testing_expctd[i, 0])
self.assertIsInstance(tst[i][1], geometry.Point)
self.assertListEqual([tst[i][1].x, tst[i][1].y],
list(testing_expctd[i, 1:]))
# change cutoff_t
cutoff_te = 0.3
vb.set_t_cutoff(cutoff_te)
# testing_expctd = self.data[self.data[:, 0] > cutoff_te]
testing_expctd = self.data[self.data[:, 0] >= cutoff_te]
self.assertEqual(vb.testing().ndata, len(testing_expctd))
self.assertTrue(np.all(vb.testing() == testing_expctd))
# testing dataset with dt_plus specified
dt_plus = testing_expctd[2, 0] - cutoff_te
# testing_expctd = testing_expctd[:3] # three results
testing_expctd = testing_expctd[:2] # two results
self.assertEqual(
vb.testing(dt_plus=dt_plus).ndata, len(testing_expctd))
self.assertTrue(np.all(vb.testing(dt_plus=dt_plus) == testing_expctd))
# testing dataset with dt_plus and dt_minus
dt_plus = self.data[self.data[:, 0] > cutoff_te][17, 0] - cutoff_te
dt_minus = self.data[self.data[:, 0] > cutoff_te][10, 0] - cutoff_te
# testing_expctd = self.data[(self.data[:, 0] > (cutoff_te + dt_minus)) & (self.data[:, 0] <= (cutoff_te + dt_plus))]
testing_expctd = self.data[(self.data[:, 0] >= (cutoff_te + dt_minus))
& (self.data[:, 0] < (cutoff_te + dt_plus))]
self.assertEqual(
vb.testing(dt_plus=dt_plus, dt_minus=dt_minus).ndata, 7)
self.assertEqual(
vb.testing(dt_plus=dt_plus, dt_minus=dt_minus).ndata,
len(testing_expctd))
self.assertTrue(
np.all(
vb.testing(dt_plus=dt_plus, dt_minus=dt_minus) ==
testing_expctd))