class TestStd(unittest.TestCase): def setUp(self): self._metric = Metric('test') self._metric.process = Mock() self._metric.process.side_effect = [torch.zeros(torch.Size([])), torch.FloatTensor([0.1, 0.2, 0.3]), torch.FloatTensor([0.4, 0.5, 0.6]), torch.FloatTensor([0.7, 0.8, 0.9]), torch.ones(torch.Size([]))] self._std = Std('test') self._std.reset({}) self._target = 0.31622776601684 def test_train(self): self._std.train() for i in range(5): self._std.process(self._metric.process()) result = self._std.process_final({}) self.assertAlmostEqual(self._target, result) def test_validate(self): self._std.eval() for i in range(5): self._std.process(self._metric.process()) result = self._std.process_final({}) self.assertAlmostEqual(self._target, result)
def test_main_loop_metrics(self): metric = Metric('test') metric.process = Mock(return_value={'test': 0}) metric.process_final = Mock(return_value={'test': 0}) metric.reset = Mock(return_value=None) data = [(torch.Tensor([1]), torch.Tensor([1])), (torch.Tensor([2]), torch.Tensor([2])), (torch.Tensor([3]), torch.Tensor([3]))] generator = DataLoader(data) train_steps = len(data) epochs = 1 callback = MagicMock() torchmodel = MagicMock() torchmodel.forward = Mock(return_value=1) optimizer = MagicMock() loss = torch.tensor([2], requires_grad=True) criterion = Mock(return_value=loss) torchbearermodel = Model(torchmodel, optimizer, criterion, [metric]) torchbearerstate = torchbearermodel.fit_generator(generator, train_steps, epochs, 0, [callback], initial_epoch=0, pass_state=False) torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].reset.assert_called_once() self.assertTrue(torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].process.call_count == len(data)) torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].process_final.assert_called_once() self.assertTrue(torchbearerstate[torchbearer.METRICS]['test'] == 0)
def test_test_loop_metrics(self): metric = Metric('test') metric.process = Mock(return_value={'test': 0}) metric.process_final = Mock(return_value={'test': 0}) metric.reset = Mock(return_value=None) metric_list = MetricList([metric]) data = [(torch.Tensor([1]), torch.Tensor([1])), (torch.Tensor([2]), torch.Tensor([2])), (torch.Tensor([3]), torch.Tensor([3]))] validation_generator = DataLoader(data) validation_steps = len(data) callback = MagicMock() callback_List = torchbearer.CallbackList([callback]) torchmodel = MagicMock() torchmodel.forward = Mock(return_value=1) optimizer = MagicMock() criterion = Mock(return_value=2) torchbearermodel = Model(torchmodel, optimizer, criterion, [metric]) state = torchbearermodel.main_state.copy() state.update({torchbearer.METRIC_LIST: metric_list, torchbearer.VALIDATION_GENERATOR: validation_generator, torchbearer.CallbackList: callback_List, torchbearer.MODEL: torchmodel, torchbearer.VALIDATION_STEPS: validation_steps, torchbearer.CRITERION: criterion, torchbearer.STOP_TRAINING: False, torchbearer.METRICS: {}}) torchbearerstate = torchbearermodel._test_loop(state, callback_List, False, Model._load_batch_standard, num_steps=None) torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].reset.assert_called_once() self.assertTrue(torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].process.call_count == len(data)) torchbearerstate[torchbearer.METRIC_LIST].metric_list[0].process_final.assert_called_once() self.assertTrue(torchbearerstate[torchbearer.METRICS]['test'] == 0)
def test_process_final(self): my_mock = Metric('test') my_mock.process_final = Mock(return_value={'test': -1}) metric = MetricList([my_mock]) result = metric.process_final({'state': -1}) self.assertEqual({'test': -1}, result) my_mock.process_final.assert_called_once_with({'state': -1})
class TestStd(unittest.TestCase): def setUp(self): self._metric = Metric('test') self._metric.process = Mock() self._metric.process.side_effect = [torch.zeros(torch.Size([])), torch.FloatTensor([0.1, 0.2, 0.3]), torch.FloatTensor([0.4, 0.5, 0.6]), torch.FloatTensor([0.7, 0.8, 0.9]), torch.ones(torch.Size([]))] self._std = Std('test') self._std.reset({}) self._target = 0.31622776601684 def test_train(self): self.setUp() self._std.train() for i in range(5): self._std.process(self._metric.process()) result = self._std.process_final({}) self.assertAlmostEqual(self._target, result, places=5) def test_validate(self): self.setUp() self._std.eval() for i in range(5): self._std.process(self._metric.process()) result = self._std.process_final({}) self.assertAlmostEqual(self._target, result, places=5) def test_precision_error(self): self.setUp() self._std.train() val = torch.tensor([0.55]) for i in range(2): self._std.process(val) result = self._std.process_final({}) self.assertEqual(0, result) def setUpMoreDims(self): self._metric = Metric('test') self._metric.process = Mock() self._metric.process.side_effect = [torch.zeros(torch.Size([])), torch.FloatTensor([[0.1, 0.2, 0.3], [1.1, 1.2, 1.3]]), torch.FloatTensor([[0.4, 0.5, 0.6], [1.4, 1.5, 1.6]]), torch.FloatTensor([[0.7, 0.8, 0.9], [1.7, 1.8, 1.9]]), torch.ones(torch.Size([]))] self._std = Std('test') self._std.reset({}) self._target = 0.57662804083742 def test_more_dims(self): self.setUpMoreDims() for i in range(5): self._std.process(self._metric.process()) result = self._std.process_final({}) self.assertAlmostEqual(self._target, result, places=5)
def setUpMoreDims(self): self._metric = Metric('test') self._metric.process = Mock() self._metric.process.side_effect = [torch.zeros(torch.Size([])), torch.FloatTensor([[0.1, 0.2, 0.3], [1.1, 1.2, 1.3]]), torch.FloatTensor([[0.4, 0.5, 0.6], [1.4, 1.5, 1.6]]), torch.FloatTensor([[0.7, 0.8, 0.9], [1.7, 1.8, 1.9]]), torch.ones(torch.Size([]))] self._mean = Mean('test') self._mean.reset({}) self._target = 0.95
def setUp(self): self._metric = Metric('test') self._metric.process = Mock() self._metric.process.side_effect = [torch.zeros(torch.Size([])), torch.FloatTensor([0.1, 0.2, 0.3]), torch.FloatTensor([0.4, 0.5, 0.6]), torch.FloatTensor([0.7, 0.8, 0.9]), torch.ones(torch.Size([]))] self._mean = Mean('test') self._mean.reset({}) self._target = 0.5
def setUpMoreDims(self): self._metric = Metric('test') self._metric.process = Mock() self._metric.process.side_effect = [ torch.zeros(torch.Size([])), torch.FloatTensor([[0.1, 0.2, 0.3], [1.1, 1.2, 1.3]]), torch.FloatTensor([[0.4, 0.5, 0.6], [1.4, 1.5, 1.6]]), torch.FloatTensor([[0.7, 0.8, 0.9], [1.7, 1.8, 1.9]]), torch.ones(torch.Size([])) ] self._std = Std('test', unbiased=False) self._std.reset({}) self._target = 0.57662804083742
def setUp(self): self._metric = Metric('test') self._metric.process = Mock() self._metric.process.side_effect = [ torch.zeros(torch.Size([])), torch.FloatTensor([0.1, 0.2, 0.3]), torch.FloatTensor([0.4, 0.5, 0.6]), torch.FloatTensor([0.7, 0.8, 0.9]), torch.ones(torch.Size([])) ] self._std = Std('test', unbiased=False) self._std.reset({}) self._target = 0.31622776601684
class TestMean(unittest.TestCase): def setUp(self): self._metric = Metric('test') self._metric.process = Mock() self._metric.process.side_effect = [torch.zeros(torch.Size([])), torch.FloatTensor([0.1, 0.2, 0.3]), torch.FloatTensor([0.4, 0.5, 0.6]), torch.FloatTensor([0.7, 0.8, 0.9]), torch.ones(torch.Size([]))] self._mean = Mean('test') self._mean.reset({}) self._target = 0.5 def test_train_dict(self): self.setUp() self._mean.train() for i in range(5): self._mean.process(self._metric.process()) result = self._mean.process_final({}) self.assertAlmostEqual(self._target, result, places=5) def test_validate_dict(self): self.setUp() self._mean.eval() for i in range(5): self._mean.process(self._metric.process()) result = self._mean.process_final({}) self.assertAlmostEqual(self._target, result, places=5) def setUpMoreDims(self): self._metric = Metric('test') self._metric.process = Mock() self._metric.process.side_effect = [torch.zeros(torch.Size([])), torch.FloatTensor([[0.1, 0.2, 0.3], [1.1, 1.2, 1.3]]), torch.FloatTensor([[0.4, 0.5, 0.6], [1.4, 1.5, 1.6]]), torch.FloatTensor([[0.7, 0.8, 0.9], [1.7, 1.8, 1.9]]), torch.ones(torch.Size([]))] self._mean = Mean('test') self._mean.reset({}) self._target = 0.95 def test_more_dims(self): self.setUpMoreDims() for i in range(5): self._mean.process(self._metric.process()) result = self._mean.process_final({}) self.assertAlmostEqual(self._target, result, places=5)
def setUp(self): self._metric = Metric('test') self._mean = RunningMean('test') self._cache = [ torch.Tensor([1.0]), torch.Tensor([1.5]), torch.Tensor([2.0]) ] self._target = 1.5
def setUp(self): self._metric = Metric('test') self._metric.train = Mock() self._metric.eval = Mock() self._metric.reset = Mock() self._metric.process = Mock(return_value='process') self._metric.process_final = Mock(return_value='process_final') self._to_dict = ToDict(self._metric)
def test_process_final(self): root = Metric('test') root.process_final = Mock(return_value='test') leaf1 = Metric('test') leaf1.process_final = Mock(return_value={'test': 10}) leaf2 = Metric('test') leaf2.process_final = Mock(return_value=None) tree = MetricTree(root) tree.add_child(leaf1) tree.add_child(leaf2) self.assertTrue(tree.process_final('args') == {'test': 10}) root.process_final.assert_called_once_with('args') leaf1.process_final.assert_called_once_with('test') leaf2.process_final.assert_called_once_with('test')
def test_dict_return(self): root = Metric('test') root.process = Mock(return_value={0: 'test', 1: 'something else'}) leaf1 = Metric('test') leaf1.process = Mock(return_value={'test': 10}) leaf2 = Metric('test') leaf2.process = Mock(return_value=None) tree = MetricTree(root) tree.add_child(leaf1) tree.add_child(leaf2) self.assertTrue(tree.process('args') == {'test': 10}) root.process.assert_called_once_with('args') leaf1.process.assert_called_once_with('test') leaf2.process.assert_called_once_with('test')
def test_reset(self): root = Metric('test') root.reset = Mock() leaf = Metric('test') leaf.reset = Mock() tree = MetricTree(root) tree.add_child(leaf) tree.reset({}) root.reset.assert_called_once_with({}) leaf.reset.assert_called_once_with({})
def test_eval(self): root = Metric('test') root.eval = Mock() leaf = Metric('test') leaf.eval = Mock() tree = MetricTree(root) tree.add_child(leaf) tree.eval() root.eval.assert_called_once() leaf.eval.assert_called_once()
def test_train(self): root = Metric('test') root.train = Mock() leaf = Metric('test') leaf.train = Mock() tree = MetricTree(root) tree.add_child(leaf) tree.train() root.train.assert_called_once() leaf.train.assert_called_once()
def test_train(self): root = Metric('test') root.train = Mock() leaf = Metric('test') leaf.train = Mock() tree = MetricTree(root) tree.add_child(leaf) tree.train() self.assertEqual(root.train.call_count, 1) self.assertEqual(leaf.train.call_count, 1)
def test_eval(self): root = Metric('test') root.eval = Mock() leaf = Metric('test') leaf.eval = Mock() tree = MetricTree(root) tree.add_child(leaf) tree.eval() self.assertEqual(root.eval.call_count, 1) self.assertEqual(leaf.eval.call_count, 1)
def setUp(self): self._metric = Metric('test') self._mean = RunningMean('test') self._cache = [1.0, 1.5, 2.0] self._target = 1.5
def test_eval(self): my_mock = Metric('test') my_mock.eval = Mock(return_value=None) metric = MetricList([my_mock]) metric.eval() self.assertEqual(my_mock.eval.call_count, 1)
def test_reset(self): my_mock = Metric('test') my_mock.reset = Mock(return_value=None) metric = MetricList([my_mock]) metric.reset({'state': -1}) my_mock.reset.assert_called_once_with({'state': -1})
def test_eval(self): my_mock = Metric('test') my_mock.eval = Mock(return_value=None) metric = MetricList([my_mock]) metric.eval() my_mock.eval.assert_called_once()
def test_train(self): my_mock = Metric('test') my_mock.train = Mock(return_value=None) metric = MetricList([my_mock]) metric.train() my_mock.train.assert_called_once()
def test_train(self): my_mock = Metric('test') my_mock.train = Mock(return_value=None) metric = MetricList([my_mock]) metric.train() self.assertEqual(my_mock.train.call_count, 1)
def test_string(self): root = Metric('test') tree = MetricTree(root) self.assertEqual(str(root), str(tree))