def eval(self):
"""
Predict and calculate metrics on both train dataloader and validation dataloader.
(train dataloader is optional for this method)
:return: Evaluation dict - Relevant features are ['train', 'val'] which are
ignite.State who contains: <br>
- result['train'].metrics['accuracy'] <br>
- result['train'].metrics['avg_loss'] <br>
- result['val'].metrics['accuracy'] <br>
- result['val'].metrics['avg_loss']
:rtype: dict
"""
validator = self.setup_evaluator()
result = dict()
self.model.eval()
with torch.no_grad():
if self.train_dloader is not None:
start = timeit.default_timer()
results_train = validator.run(self.train_dloader, max_epochs=1, epoch_length=self.eval_steps)
end = timeit.default_timer()
results_train.eval_duration_seconds = end - start
result['train'] = results_train
tg.reset()
start = timeit.default_timer()
results_val = validator.run(self.val_dloader, max_epochs=1, epoch_length=self.eval_steps)
end = timeit.default_timer()
tg.reset()
results_val.eval_duration_seconds = end - start
result['val'] = results_val
self.model.train()
if self.train_dloader is not None:
print('Train accuracy:', results_train.metrics['accuracy'])
print('Test accuracy:', results_val.metrics['accuracy'])
return result
def test_matches_ignores_spaces_global():
import tensorguard as tg
tg.reset()
a = np.ones([1, 2, 3])
assert tg.matches(a, "1,2,3")
assert tg.matches(a, "1 , 2, 3 ")
assert tg.matches(a, "1, 2,3 ")
def test_get_dims():
import tensorguard as tg
tg.reset()
assert tg.get_dims() == {}
x = np.zeros([15, 4, 32])
tg.guard(x, "B, C, W")
assert tg.get_dims("B * 2, W/4") == [30, 8]
def test_reset():
import tensorguard as tg
tg.reset()
tg.guard([3, 5, 4], "A, B, C")
assert tg.get_dims() == {"A": 3, "B": 5, "C": 4}
tg.reset()
assert tg.get_dims() == {}
def test_guard_raises_inferred_global():
import tensorguard as tg
tg.reset()
a = np.ones([1, 2, 3])
b = np.ones([3, 2, 5])
tg.guard(a, "A, B, C")
with pytest.raises(ShapeError):
tg.guard(b, "C, B, A")
def test_matches_wildcards_global():
import tensorguard as tg
tg.reset()
z = np.ones([1, 2, 4, 8])
assert tg.matches(z, "1, 2, 4, *")
assert tg.matches(z, "*, *, *, 8")
assert not tg.matches(z, "*")
assert not tg.matches(z, "*, *, *")
def test_matches_basic_numerical_global():
import tensorguard as tg
tg.reset()
a = np.ones([1, 2, 3])
assert tg.matches(a, "1, 2, 3")
assert not tg.matches(a, "1, 2, 4")
assert not tg.matches(a, "1, 2, 3, 4")
assert not tg.matches(a, "1, 2")
def test_guard_dynamic_shape_global():
import tensorguard as tg
tg.reset()
with pytest.raises(ShapeError):
tg.guard([None, 2, 3], "C, B, A")
tg.guard([None, 2, 3], "?, B, A")
tg.guard([1, 2, 3], "C?, B, A")
tg.guard([None, 2, 3], "C?, B, A")
def test_get_dim():
import tensorguard as tg
tg.reset()
x = np.zeros([32, 2, 5])
tg.guard(x, "B, C, W")
assert tg.get_dim("W") == 5
with pytest.raises(KeyError):
tg.get_dim("W_FAKE")
assert tg.safe_get_dim("W_FAKE") is None
def test_guard_infers_dimensions_complex_global():
import tensorguard as tg
tg.reset()
a = np.ones([1, 2, 3])
tg.guard(a, "A, B*2, A+C")
assert tg.get_dims() == {
"A": 1,
"B": 1,
"C": 2
}, f'{tg.get_dims()}' + ' != {"A": 1, "B": 1, "C": 2}'
def test_set_dim():
import tensorguard as tg
tg.reset()
x = np.zeros([32, 2, 5])
tg.guard(x, "B, C, W")
assert tg.get_dim("W") == 5
tg.set_dim("W", 10)
assert tg.get_dim("W") == 10
tg.set_dim("WF", 40)
assert tg.get_dim("WF") == 40
assert tg.safe_get_dim("W_FAKE") is None
tg.set_dims(WW=32, HH=55)
assert tg.get_dim("WW") == 32
assert tg.safe_get_dim("HH") == 55
def test_guard_ellipsis_global():
import tensorguard as tg
tg.reset()
a = np.ones([1, 2, 3, 4, 5])
tg.guard(a, "...")
tg.guard(a, "..., 5")
tg.guard(a, "..., 4, 5")
tg.guard(a, "1, ...")
tg.guard(a, "1, 2, ...")
tg.guard(a, "1, 2, ..., 4, 5")
tg.guard(a, "1, 2, 3, ..., 4, 5")
with pytest.raises(ShapeError):
tg.guard(a, "1, 2, 3, 4, 5, 6,...")
with pytest.raises(ShapeError):
tg.guard(a, "..., 1, 2, 3, 4, 5, 6")
def test_guard_ignores_wildcard_global():
import tensorguard as tg
tg.reset()
a = np.ones([1, 2, 3])
tg.guard(a, "*, *, 3")
assert tg.get_dims() == {}
def test_guard_raises_complex_global():
import tensorguard as tg
tg.reset()
a = np.ones([1, 2, 3])
with pytest.raises(ShapeError):
tg.guard(a, "A, B, B")
def test_guard_infers_dimensions_operator_priority_global():
import tensorguard as tg
tg.reset()
a = np.ones([1, 2, 8])
tg.guard(a, "A, B, A+C*2+1")
assert tg.get_dims() == {"A": 1, "B": 2, "C": 3}
def test_guard_ellipsis_infer_dims_global():
import tensorguard as tg
tg.reset()
a = np.ones([1, 2, 3, 4, 5])
tg.guard(a, "A, B, ..., C")
assert tg.get_dims() == {"A": 1, "B": 2, "C": 5}
def test_guard_infers_dimensions_global():
import tensorguard as tg
tg.reset()
a = np.ones([1, 2, 3])
tg.guard(a, "A, B, C")
assert tg.get_dims() == {"A": 1, "B": 2, "C": 3}
