def test_save_performance_project_types():
from sasctl.tasks import update_model_performance
with mock.patch('sasctl._services.model_repository.ModelRepository.get_model') as model:
with mock.patch('sasctl._services.model_repository.ModelRepository.get_project') as project:
model.return_value = RestObj(name='fakemodel', projectId=1)
# Function is required
with pytest.raises(ValueError):
project.return_value = {}
update_model_performance(None, None, None)
# Target Level is required
with pytest.raises(ValueError):
project.return_value = {'function': 'Prediction'}
update_model_performance(None, None, None)
# Prediction variable required
with pytest.raises(ValueError):
project.return_value = {'function': 'Prediction',
'targetLevel': 'Binary'}
update_model_performance(None, None, None)
# Classification variable required
with pytest.raises(ValueError):
project.return_value = {'function': 'classification',
'targetLevel': 'Binary'}
update_model_performance(None, None, None)
def test_update_model_performance(self, sklearn_linear_model, cas_session):
from six.moves import mock
from sasctl.tasks import update_model_performance
lm, X, y = sklearn_linear_model
# Score & set output var
train_df = X.copy()
train_df['var1'] = lm.predict(X)
train_df['Price'] = y
with mock.patch('swat.CAS') as CAS:
for period in ('q12019', 'q22019', 'q32019', 'q42019'):
sample = train_df.sample(frac=0.1)
update_model_performance(sample, self.MODEL_NAME, period)
def test_update_model_performance(self, sklearn_linear_model, cas_session):
from six.moves import mock
from sasctl.tasks import update_model_performance
lm, X, y = sklearn_linear_model
# Score & set output var
train_df = X.copy()
train_df['var1'] = lm.predict(X)
train_df['Price'] = y
with mock.patch('swat.CAS') as CAS:
CAS.return_value = cas_session
# NOTE: can only automate testing of 1 period at a time since
# upload_model_performance closes the CAS session when it's done.
for period in ['q12019']:
sample = train_df.sample(frac=0.1)
update_model_performance(sample, self.MODEL_NAME, period)
# Call the published module and score the record
result = module_lm.score(**x)
print(result)
# Build a second model
dt = DecisionTreeRegressor()
dt.fit(X_train, y_train)
# Register the second model in Model Manager
model_dt = register_model(dt, 'Decision Tree', project, input=X)
# Publish from Model Manager -> MAS
module_dt = publish_model(model_dt, 'maslocal')
# Use MAS to score some new data
result = module_dt.score(**x)
print(result)
# Model Manager can track model performance over time if provided with
# historical model observations & predictions. SIMULATE historical data by
# repeatedly sampling from the test set.
perf_df = X_test.copy()
perf_df['var1'] = lm.predict(X_test)
perf_df['Price'] = y
# For each (simulated) historical period, upload model results
for period in ('q12019', 'q22019', 'q32019', 'q42019'):
sample = perf_df.sample(frac=0.2)
update_model_performance(sample, model_name, period)