def test_persist_if_better_than_no_dataset_test(self, fit, dataset_loader):
model, model_persister = Mock(), Mock()
model.score.return_value = 0.9
with pytest.raises(ValueError):
fit(
dataset_loader_train=dataset_loader,
model=model,
model_persister=model_persister,
dataset_loader_test=None,
persist_if_better_than=0.9,
)
def test_persist_if_better_than_no_dataset_test(self, fit, dataset_loader):
model, model_persister = Mock(), Mock()
model.score.return_value = 0.9
with pytest.raises(ValueError):
fit(
dataset_loader_train=dataset_loader,
model=model,
model_persister=model_persister,
dataset_loader_test=None,
persist_if_better_than=0.9,
)
def test_evaluate_no_score(self, fit, dataset_loader):
model = Mock()
del model.score
del model.cv_results_
with pytest.raises(ValueError):
fit(
dataset_loader_train=dataset_loader,
model=model,
model_persister=Mock(),
dataset_loader_test=dataset_loader,
evaluate=True,
)
def test_evaluate_scoring(self, fit, dataset_loader):
model = Mock()
del model.cv_results_
scorer = Mock()
scorer.side_effect = [0.99, 0.01]
fit(
dataset_loader_train=dataset_loader,
model=model,
model_persister=Mock(),
dataset_loader_test=dataset_loader,
scoring=scorer,
evaluate=True,
)
assert model.score.call_count == 0
assert scorer.call_count == 2
def test_it(self, fit):
model, dataset_loader_train, model_persister = Mock(), Mock(), Mock()
X, y = object(), object()
dataset_loader_train.return_value = X, y
result = fit(dataset_loader_train, model, model_persister)
assert result is model
dataset_loader_train.assert_called_with()
model.fit.assert_called_with(X, y)
model_persister.write.assert_called_with(model)
def test_it(self, fit):
model, dataset_loader_train, model_persister = Mock(), Mock(), Mock()
X, y = object(), object()
dataset_loader_train.return_value = X, y
result = fit(dataset_loader_train, model, model_persister)
assert result is model
dataset_loader_train.assert_called_with()
model.fit.assert_called_with(X, y)
model_persister.write.assert_called_with(model)
def test_activate_no_persist(self, fit, dataset_loader):
model, model_persister = Mock(), Mock()
result = fit(
dataset_loader_train=dataset_loader,
model=model,
model_persister=model_persister,
persist=False,
)
assert result is model
model_persister.activate.call_count == 0
def test_activate_no_persist(self, fit, dataset_loader):
model, model_persister = Mock(), Mock()
result = fit(
dataset_loader_train=dataset_loader,
model=model,
model_persister=model_persister,
persist=False,
)
assert result is model
model_persister.activate.call_count == 0
def test_no_persist(self, fit):
model, dataset_loader_train, model_persister = Mock(), Mock(), Mock()
X, y = object(), object()
dataset_loader_train.return_value = X, y
result = fit(dataset_loader_train, model, model_persister,
persist=False)
assert result is model
dataset_loader_train.assert_called_with()
model.fit.assert_called_with(X, y)
assert model_persister.call_count == 0
def test_evaluate_no_test_dataset(self, fit):
model, dataset_loader_train, model_persister = Mock(), Mock(), Mock()
X, y = object(), object()
dataset_loader_train.return_value = X, y
result = fit(dataset_loader_train, model, model_persister,
evaluate=True)
assert result is model
dataset_loader_train.assert_called_with()
model.fit.assert_called_with(X, y)
assert model.score.call_count == 1
model.score.assert_called_with(X, y)
model_persister.write.assert_called_with(model)
def test_persist_if_better_than(self, fit, dataset_loader):
model, model_persister = Mock(), Mock()
model.score.return_value = 0.9
result = fit(
dataset_loader_train=dataset_loader,
model=model,
model_persister=model_persister,
dataset_loader_test=dataset_loader,
persist_if_better_than=0.9,
)
assert result is model
assert model_persister.write.call_count == 1
def test_evaluate_annotations(self, fit, dataset_loader):
model = Mock()
model.score.side_effect = [0.9, 0.8]
result = fit(
dataset_loader_train=dataset_loader,
model=model,
model_persister=Mock(),
dataset_loader_test=dataset_loader,
persist_if_better_than=0.9,
)
assert result.__metadata__['score_train'] == 0.9
assert result.__metadata__['score_test'] == 0.8
def test_no_persist(self, fit):
model, dataset_loader_train, model_persister = Mock(), Mock(), Mock()
X, y = object(), object()
dataset_loader_train.return_value = X, y
result = fit(dataset_loader_train,
model,
model_persister,
persist=False)
assert result is model
dataset_loader_train.assert_called_with()
model.fit.assert_called_with(X, y)
assert model_persister.call_count == 0
def test_evaluate_annotations(self, fit, dataset_loader):
model = Mock()
model.score.side_effect = [0.9, 0.8]
result = fit(
dataset_loader_train=dataset_loader,
model=model,
model_persister=Mock(),
dataset_loader_test=dataset_loader,
persist_if_better_than=0.9,
)
assert result.__metadata__['score_train'] == 0.9
assert result.__metadata__['score_test'] == 0.8
def test_persist_if_better_than_false(self, fit, dataset_loader):
model, model_persister = Mock(), Mock()
model.score.return_value = 0.9
result = fit(
dataset_loader_train=dataset_loader,
model=model,
model_persister=model_persister,
dataset_loader_test=dataset_loader,
persist_if_better_than=0.91,
)
assert result is model
assert model_persister.write.call_count == 0
def test_evaluate_no_test_dataset(self, fit):
model, dataset_loader_train, model_persister = Mock(), Mock(), Mock()
X, y = object(), object()
dataset_loader_train.return_value = X, y
result = fit(dataset_loader_train,
model,
model_persister,
evaluate=True)
assert result is model
dataset_loader_train.assert_called_with()
model.fit.assert_called_with(X, y)
assert model.score.call_count == 1
model.score.assert_called_with(X, y)
model_persister.write.assert_called_with(model)
def test_evaluate_with_test_dataset(self, fit):
model, dataset_loader_train, model_persister = Mock(), Mock(), Mock()
dataset_loader_test = Mock()
X, y, X_test, y_test = object(), object(), object(), object()
dataset_loader_train.return_value = X, y
dataset_loader_test.return_value = X_test, y_test
result = fit(dataset_loader_train, model, model_persister,
dataset_loader_test=dataset_loader_test,
evaluate=True)
assert result is model
dataset_loader_train.assert_called_with()
dataset_loader_test.assert_called_with()
model.fit.assert_called_with(X, y)
assert model.score.call_count == 2
assert model.score.mock_calls[0] == call(X, y)
assert model.score.mock_calls[1] == call(X_test, y_test)
model_persister.write.assert_called_with(model)
def test_timestamp(self, fit, dataset_loader):
model, model_persister = Mock(), Mock()
def persist(model):
assert 'train_timestamp' in model.__metadata__
model_persister.write.side_effect = persist
before_fit = datetime.now()
result = fit(
dataset_loader,
model,
model_persister,
)
after_fit = datetime.now()
assert result is model
timestamp = parse(model.__metadata__['train_timestamp'])
assert before_fit < timestamp < after_fit
model_persister.write.assert_called_with(model)
def test_evaluate_with_test_dataset(self, fit):
model, dataset_loader_train, model_persister = Mock(), Mock(), Mock()
dataset_loader_test = Mock()
X, y, X_test, y_test = object(), object(), object(), object()
dataset_loader_train.return_value = X, y
dataset_loader_test.return_value = X_test, y_test
result = fit(dataset_loader_train,
model,
model_persister,
dataset_loader_test=dataset_loader_test,
evaluate=True)
assert result is model
dataset_loader_train.assert_called_with()
dataset_loader_test.assert_called_with()
model.fit.assert_called_with(X, y)
assert model.score.call_count == 2
assert model.score.mock_calls[0] == call(X, y)
assert model.score.mock_calls[1] == call(X_test, y_test)
model_persister.write.assert_called_with(model)
def test_timestamp(self, fit, dataset_loader):
model, model_persister = Mock(), Mock()
def persist(model):
assert 'train_timestamp' in model.__metadata__
model_persister.write.side_effect = persist
before_fit = datetime.now()
result = fit(
dataset_loader,
model,
model_persister,
)
after_fit = datetime.now()
assert result is model
timestamp = parse(model.__metadata__['train_timestamp'])
assert before_fit < timestamp < after_fit
model_persister.write.assert_called_with(model)
def test_cv_results(self, fit, dataset_loader):
model, model_persister = Mock(), Mock()
model.cv_results_ = {
'mean_train_score': [3, 2, 1],
'mean_test_score': [1, 2, 3],
}
def persist(model):
assert 'cv_results' in model.__metadata__
model_persister.write.side_effect = persist
result = fit(
dataset_loader,
model,
model_persister,
)
assert result is model
cv_results = model.__metadata__['cv_results']
cv_results = pandas.read_json(cv_results).to_dict(orient='list')
assert cv_results == model.cv_results_
model_persister.write.assert_called_with(model)
