def fold_request(cv_opts):
"""Computes k-fold cross-validation folds.
Emulates :func:`optunity.cross_validated`."""
try:
num_instances = cv_opts['num_instances']
except (KeyError, ValueError):
msg = {'error_msg': 'number of instances num_instances must be set.'}
comm.send(comm.json_encode(msg))
print(startup_msg, file=sys.stderr)
exit(1)
num_folds = cv_opts.get('num_folds', 10)
strata = cv_opts.get('strata', None)
clusters = cv_opts.get('clusters', None)
num_iter = cv_opts.get('num_iter', 1)
folds = [
optunity.generate_folds(num_instances,
num_folds=num_folds,
strata=strata,
clusters=clusters) for _ in range(num_iter)
]
msg = {'folds': folds}
comm.send(comm.json_encode(msg))
exit(0)
def fold_request(cv_opts):
"""Computes k-fold cross-validation folds.
Emulates :func:`optunity.cross_validated`."""
try:
num_instances = cv_opts['num_instances']
except (KeyError, ValueError):
msg = {'error_msg': 'number of instances num_instances must be set.'}
comm.send(comm.json_encode(msg))
print(startup_msg, file=sys.stderr)
exit(1)
num_folds = cv_opts.get('num_folds', 10)
strata = cv_opts.get('strata', None)
clusters = cv_opts.get('clusters', None)
num_iter = cv_opts.get('num_iter', 1)
folds = [optunity.generate_folds(num_instances, num_folds=num_folds,
strata=strata, clusters=clusters)
for _ in range(num_iter)]
msg = {'folds': folds}
comm.send(comm.json_encode(msg))
exit(0)
tuned_model = sklearn.svm.SVR(**optimal_pars).fit(x_train, y_train)
predictions = tuned_model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# decorate with cross-validation
compute_mse_tuned = outer_cv(compute_mse_tuned)
t = time.time()
mse_tuned = compute_mse_tuned()
diff = time.time() - t
print('Nested cv mean squared error of tuned model: ' + str(mse_tuned))
print('Tuning time (approx): ' + str(diff/3) + ' seconds') # we tuned 3 times
# generate folds, so we know the indices of test instances at any point
folds = optunity.generate_folds(data.shape[0], num_folds=3)
# create another cross-validation decorator
# we will compare nested cross-validation results for both tuned and untuned models
# to do this, we will perform nested cross-validation but aggregate results using the identity function
# this will yield the predictions
outer_cv = optunity.cross_validated(x=data, y=targets, num_folds=3, folds=[folds],
aggregator=optunity.cross_validation.identity)
def svr_untuned_predictions(x_train, y_train, x_test, y_test):
model = sklearn.svm.SVR().fit(x_train, y_train)
return model.predict(x_test).tolist()
def svr_tuned_predictions(x_train, y_train, x_test, y_test):
@optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
predictions = tuned_model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# decorate with cross-validation
compute_mse_tuned = outer_cv(compute_mse_tuned)
t = time.time()
mse_tuned = compute_mse_tuned()
diff = time.time() - t
print('Nested cv mean squared error of tuned model: ' + str(mse_tuned))
print('Tuning time (approx): ' + str(diff / 3) +
' seconds') # we tuned 3 times
# generate folds, so we know the indices of test instances at any point
folds = optunity.generate_folds(data.shape[0], num_folds=3)
# create another cross-validation decorator
# we will compare nested cross-validation results for both tuned and untuned models
# to do this, we will perform nested cross-validation but aggregate results using the identity function
# this will yield the predictions
outer_cv = optunity.cross_validated(
x=data,
y=targets,
num_folds=3,
folds=[folds],
aggregator=optunity.cross_validation.identity)
def svr_untuned_predictions(x_train, y_train, x_test, y_test):
model = sklearn.svm.SVR().fit(x_train, y_train)