class TestStd(unittest.TestCase):
def setUp(self):
self._metric = Metric('test')
self._metric.process = Mock()
self._metric.process.side_effect = [torch.zeros(torch.Size([])),
torch.FloatTensor([0.1, 0.2, 0.3]),
torch.FloatTensor([0.4, 0.5, 0.6]),
torch.FloatTensor([0.7, 0.8, 0.9]),
torch.ones(torch.Size([]))]
self._std = Std('test')
self._std.reset({})
self._target = 0.31622776601684
def test_train(self):
self._std.train()
for i in range(5):
self._std.process(self._metric.process())
result = self._std.process_final({})
self.assertAlmostEqual(self._target, result)
def test_validate(self):
self._std.eval()
for i in range(5):
self._std.process(self._metric.process())
result = self._std.process_final({})
self.assertAlmostEqual(self._target, result)
def test_main_loop_metrics(self):
metric = Metric('test')
metric.process = Mock(return_value={'test': 0})
metric.process_final = Mock(return_value={'test': 0})
metric.reset = Mock(return_value=None)
data = [(torch.Tensor([1]), torch.Tensor([1])), (torch.Tensor([2]), torch.Tensor([2])), (torch.Tensor([3]), torch.Tensor([3]))]
generator = DataLoader(data)
train_steps = len(data)
epochs = 1
callback = MagicMock()
torchmodel = MagicMock()
torchmodel.forward = Mock(return_value=1)
optimizer = MagicMock()
loss = torch.tensor([2], requires_grad=True)
criterion = Mock(return_value=loss)
torchbearermodel = Model(torchmodel, optimizer, criterion, [metric])
torchbearerstate = torchbearermodel.fit_generator(generator, train_steps, epochs, 0, [callback], initial_epoch=0, pass_state=False)
torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].reset.assert_called_once()
self.assertTrue(torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].process.call_count == len(data))
torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].process_final.assert_called_once()
self.assertTrue(torchbearerstate[torchbearer.METRICS]['test'] == 0)
def test_test_loop_metrics(self):
metric = Metric('test')
metric.process = Mock(return_value={'test': 0})
metric.process_final = Mock(return_value={'test': 0})
metric.reset = Mock(return_value=None)
metric_list = MetricList([metric])
data = [(torch.Tensor([1]), torch.Tensor([1])), (torch.Tensor([2]), torch.Tensor([2])), (torch.Tensor([3]), torch.Tensor([3]))]
validation_generator = DataLoader(data)
validation_steps = len(data)
callback = MagicMock()
callback_List = torchbearer.CallbackList([callback])
torchmodel = MagicMock()
torchmodel.forward = Mock(return_value=1)
optimizer = MagicMock()
criterion = Mock(return_value=2)
torchbearermodel = Model(torchmodel, optimizer, criterion, [metric])
state = torchbearermodel.main_state.copy()
state.update({torchbearer.METRIC_LIST: metric_list, torchbearer.VALIDATION_GENERATOR: validation_generator,
torchbearer.CallbackList: callback_List, torchbearer.MODEL: torchmodel, torchbearer.VALIDATION_STEPS: validation_steps,
torchbearer.CRITERION: criterion, torchbearer.STOP_TRAINING: False, torchbearer.METRICS: {}})
torchbearerstate = torchbearermodel._test_loop(state, callback_List, False, Model._load_batch_standard, num_steps=None)
torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].reset.assert_called_once()
self.assertTrue(torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].process.call_count == len(data))
torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].process_final.assert_called_once()
self.assertTrue(torchbearerstate[torchbearer.METRICS]['test'] == 0)
def test_process_final(self):
my_mock = Metric('test')
my_mock.process_final = Mock(return_value={'test': -1})
metric = MetricList([my_mock])
result = metric.process_final({'state': -1})
self.assertEqual({'test': -1}, result)
my_mock.process_final.assert_called_once_with({'state': -1})
class TestStd(unittest.TestCase):
def setUp(self):
self._metric = Metric('test')
self._metric.process = Mock()
self._metric.process.side_effect = [torch.zeros(torch.Size([])),
torch.FloatTensor([0.1, 0.2, 0.3]),
torch.FloatTensor([0.4, 0.5, 0.6]),
torch.FloatTensor([0.7, 0.8, 0.9]),
torch.ones(torch.Size([]))]
self._std = Std('test')
self._std.reset({})
self._target = 0.31622776601684
def test_train(self):
self.setUp()
self._std.train()
for i in range(5):
self._std.process(self._metric.process())
result = self._std.process_final({})
self.assertAlmostEqual(self._target, result, places=5)
def test_validate(self):
self.setUp()
self._std.eval()
for i in range(5):
self._std.process(self._metric.process())
result = self._std.process_final({})
self.assertAlmostEqual(self._target, result, places=5)
def test_precision_error(self):
self.setUp()
self._std.train()
val = torch.tensor([0.55])
for i in range(2):
self._std.process(val)
result = self._std.process_final({})
self.assertEqual(0, result)
def setUpMoreDims(self):
self._metric = Metric('test')
self._metric.process = Mock()
self._metric.process.side_effect = [torch.zeros(torch.Size([])),
torch.FloatTensor([[0.1, 0.2, 0.3], [1.1, 1.2, 1.3]]),
torch.FloatTensor([[0.4, 0.5, 0.6], [1.4, 1.5, 1.6]]),
torch.FloatTensor([[0.7, 0.8, 0.9], [1.7, 1.8, 1.9]]),
torch.ones(torch.Size([]))]
self._std = Std('test')
self._std.reset({})
self._target = 0.57662804083742
def test_more_dims(self):
self.setUpMoreDims()
for i in range(5):
self._std.process(self._metric.process())
result = self._std.process_final({})
self.assertAlmostEqual(self._target, result, places=5)
def setUpMoreDims(self):
self._metric = Metric('test')
self._metric.process = Mock()
self._metric.process.side_effect = [torch.zeros(torch.Size([])),
torch.FloatTensor([[0.1, 0.2, 0.3], [1.1, 1.2, 1.3]]),
torch.FloatTensor([[0.4, 0.5, 0.6], [1.4, 1.5, 1.6]]),
torch.FloatTensor([[0.7, 0.8, 0.9], [1.7, 1.8, 1.9]]),
torch.ones(torch.Size([]))]
self._mean = Mean('test')
self._mean.reset({})
self._target = 0.95
def setUp(self):
self._metric = Metric('test')
self._metric.process = Mock()
self._metric.process.side_effect = [torch.zeros(torch.Size([])),
torch.FloatTensor([0.1, 0.2, 0.3]),
torch.FloatTensor([0.4, 0.5, 0.6]),
torch.FloatTensor([0.7, 0.8, 0.9]),
torch.ones(torch.Size([]))]
self._mean = Mean('test')
self._mean.reset({})
self._target = 0.5
def setUpMoreDims(self):
self._metric = Metric('test')
self._metric.process = Mock()
self._metric.process.side_effect = [
torch.zeros(torch.Size([])),
torch.FloatTensor([[0.1, 0.2, 0.3], [1.1, 1.2, 1.3]]),
torch.FloatTensor([[0.4, 0.5, 0.6], [1.4, 1.5, 1.6]]),
torch.FloatTensor([[0.7, 0.8, 0.9], [1.7, 1.8, 1.9]]),
torch.ones(torch.Size([]))
]
self._std = Std('test', unbiased=False)
self._std.reset({})
self._target = 0.57662804083742
def setUp(self):
self._metric = Metric('test')
self._metric.process = Mock()
self._metric.process.side_effect = [
torch.zeros(torch.Size([])),
torch.FloatTensor([0.1, 0.2, 0.3]),
torch.FloatTensor([0.4, 0.5, 0.6]),
torch.FloatTensor([0.7, 0.8, 0.9]),
torch.ones(torch.Size([]))
]
self._std = Std('test', unbiased=False)
self._std.reset({})
self._target = 0.31622776601684
class TestMean(unittest.TestCase):
def setUp(self):
self._metric = Metric('test')
self._metric.process = Mock()
self._metric.process.side_effect = [torch.zeros(torch.Size([])),
torch.FloatTensor([0.1, 0.2, 0.3]),
torch.FloatTensor([0.4, 0.5, 0.6]),
torch.FloatTensor([0.7, 0.8, 0.9]),
torch.ones(torch.Size([]))]
self._mean = Mean('test')
self._mean.reset({})
self._target = 0.5
def test_train_dict(self):
self.setUp()
self._mean.train()
for i in range(5):
self._mean.process(self._metric.process())
result = self._mean.process_final({})
self.assertAlmostEqual(self._target, result, places=5)
def test_validate_dict(self):
self.setUp()
self._mean.eval()
for i in range(5):
self._mean.process(self._metric.process())
result = self._mean.process_final({})
self.assertAlmostEqual(self._target, result, places=5)
def setUpMoreDims(self):
self._metric = Metric('test')
self._metric.process = Mock()
self._metric.process.side_effect = [torch.zeros(torch.Size([])),
torch.FloatTensor([[0.1, 0.2, 0.3], [1.1, 1.2, 1.3]]),
torch.FloatTensor([[0.4, 0.5, 0.6], [1.4, 1.5, 1.6]]),
torch.FloatTensor([[0.7, 0.8, 0.9], [1.7, 1.8, 1.9]]),
torch.ones(torch.Size([]))]
self._mean = Mean('test')
self._mean.reset({})
self._target = 0.95
def test_more_dims(self):
self.setUpMoreDims()
for i in range(5):
self._mean.process(self._metric.process())
result = self._mean.process_final({})
self.assertAlmostEqual(self._target, result, places=5)
def setUp(self):
self._metric = Metric('test')
self._mean = RunningMean('test')
self._cache = [
torch.Tensor([1.0]),
torch.Tensor([1.5]),
torch.Tensor([2.0])
]
self._target = 1.5
def setUp(self):
self._metric = Metric('test')
self._metric.train = Mock()
self._metric.eval = Mock()
self._metric.reset = Mock()
self._metric.process = Mock(return_value='process')
self._metric.process_final = Mock(return_value='process_final')
self._to_dict = ToDict(self._metric)
def test_process_final(self):
root = Metric('test')
root.process_final = Mock(return_value='test')
leaf1 = Metric('test')
leaf1.process_final = Mock(return_value={'test': 10})
leaf2 = Metric('test')
leaf2.process_final = Mock(return_value=None)
tree = MetricTree(root)
tree.add_child(leaf1)
tree.add_child(leaf2)
self.assertTrue(tree.process_final('args') == {'test': 10})
root.process_final.assert_called_once_with('args')
leaf1.process_final.assert_called_once_with('test')
leaf2.process_final.assert_called_once_with('test')
def test_dict_return(self):
root = Metric('test')
root.process = Mock(return_value={0: 'test', 1: 'something else'})
leaf1 = Metric('test')
leaf1.process = Mock(return_value={'test': 10})
leaf2 = Metric('test')
leaf2.process = Mock(return_value=None)
tree = MetricTree(root)
tree.add_child(leaf1)
tree.add_child(leaf2)
self.assertTrue(tree.process('args') == {'test': 10})
root.process.assert_called_once_with('args')
leaf1.process.assert_called_once_with('test')
leaf2.process.assert_called_once_with('test')
def test_reset(self):
root = Metric('test')
root.reset = Mock()
leaf = Metric('test')
leaf.reset = Mock()
tree = MetricTree(root)
tree.add_child(leaf)
tree.reset({})
root.reset.assert_called_once_with({})
leaf.reset.assert_called_once_with({})
def test_eval(self):
root = Metric('test')
root.eval = Mock()
leaf = Metric('test')
leaf.eval = Mock()
tree = MetricTree(root)
tree.add_child(leaf)
tree.eval()
root.eval.assert_called_once()
leaf.eval.assert_called_once()
def test_train(self):
root = Metric('test')
root.train = Mock()
leaf = Metric('test')
leaf.train = Mock()
tree = MetricTree(root)
tree.add_child(leaf)
tree.train()
root.train.assert_called_once()
leaf.train.assert_called_once()
def test_train(self):
root = Metric('test')
root.train = Mock()
leaf = Metric('test')
leaf.train = Mock()
tree = MetricTree(root)
tree.add_child(leaf)
tree.train()
self.assertEqual(root.train.call_count, 1)
self.assertEqual(leaf.train.call_count, 1)
def test_eval(self):
root = Metric('test')
root.eval = Mock()
leaf = Metric('test')
leaf.eval = Mock()
tree = MetricTree(root)
tree.add_child(leaf)
tree.eval()
self.assertEqual(root.eval.call_count, 1)
self.assertEqual(leaf.eval.call_count, 1)
def setUp(self):
self._metric = Metric('test')
self._mean = RunningMean('test')
self._cache = [1.0, 1.5, 2.0]
self._target = 1.5
def test_eval(self):
my_mock = Metric('test')
my_mock.eval = Mock(return_value=None)
metric = MetricList([my_mock])
metric.eval()
self.assertEqual(my_mock.eval.call_count, 1)
def test_reset(self):
my_mock = Metric('test')
my_mock.reset = Mock(return_value=None)
metric = MetricList([my_mock])
metric.reset({'state': -1})
my_mock.reset.assert_called_once_with({'state': -1})
def test_eval(self):
my_mock = Metric('test')
my_mock.eval = Mock(return_value=None)
metric = MetricList([my_mock])
metric.eval()
my_mock.eval.assert_called_once()
def test_train(self):
my_mock = Metric('test')
my_mock.train = Mock(return_value=None)
metric = MetricList([my_mock])
metric.train()
my_mock.train.assert_called_once()
def test_train(self):
my_mock = Metric('test')
my_mock.train = Mock(return_value=None)
metric = MetricList([my_mock])
metric.train()
self.assertEqual(my_mock.train.call_count, 1)
def test_string(self):
root = Metric('test')
tree = MetricTree(root)
self.assertEqual(str(root), str(tree))