def fold_request(cv_opts): """Computes k-fold cross-validation folds. Emulates :func:`optunity.cross_validated`.""" try: num_instances = cv_opts['num_instances'] except (KeyError, ValueError): msg = {'error_msg': 'number of instances num_instances must be set.'} comm.send(comm.json_encode(msg)) print(startup_msg, file=sys.stderr) exit(1) num_folds = cv_opts.get('num_folds', 10) strata = cv_opts.get('strata', None) clusters = cv_opts.get('clusters', None) num_iter = cv_opts.get('num_iter', 1) folds = [ optunity.generate_folds(num_instances, num_folds=num_folds, strata=strata, clusters=clusters) for _ in range(num_iter) ] msg = {'folds': folds} comm.send(comm.json_encode(msg)) exit(0)
def fold_request(cv_opts): """Computes k-fold cross-validation folds. Emulates :func:`optunity.cross_validated`.""" try: num_instances = cv_opts['num_instances'] except (KeyError, ValueError): msg = {'error_msg': 'number of instances num_instances must be set.'} comm.send(comm.json_encode(msg)) print(startup_msg, file=sys.stderr) exit(1) num_folds = cv_opts.get('num_folds', 10) strata = cv_opts.get('strata', None) clusters = cv_opts.get('clusters', None) num_iter = cv_opts.get('num_iter', 1) folds = [optunity.generate_folds(num_instances, num_folds=num_folds, strata=strata, clusters=clusters) for _ in range(num_iter)] msg = {'folds': folds} comm.send(comm.json_encode(msg)) exit(0)
tuned_model = sklearn.svm.SVR(**optimal_pars).fit(x_train, y_train) predictions = tuned_model.predict(x_test) return optunity.metrics.mse(y_test, predictions) # decorate with cross-validation compute_mse_tuned = outer_cv(compute_mse_tuned) t = time.time() mse_tuned = compute_mse_tuned() diff = time.time() - t print('Nested cv mean squared error of tuned model: ' + str(mse_tuned)) print('Tuning time (approx): ' + str(diff/3) + ' seconds') # we tuned 3 times # generate folds, so we know the indices of test instances at any point folds = optunity.generate_folds(data.shape[0], num_folds=3) # create another cross-validation decorator # we will compare nested cross-validation results for both tuned and untuned models # to do this, we will perform nested cross-validation but aggregate results using the identity function # this will yield the predictions outer_cv = optunity.cross_validated(x=data, y=targets, num_folds=3, folds=[folds], aggregator=optunity.cross_validation.identity) def svr_untuned_predictions(x_train, y_train, x_test, y_test): model = sklearn.svm.SVR().fit(x_train, y_train) return model.predict(x_test).tolist() def svr_tuned_predictions(x_train, y_train, x_test, y_test): @optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
predictions = tuned_model.predict(x_test) return optunity.metrics.mse(y_test, predictions) # decorate with cross-validation compute_mse_tuned = outer_cv(compute_mse_tuned) t = time.time() mse_tuned = compute_mse_tuned() diff = time.time() - t print('Nested cv mean squared error of tuned model: ' + str(mse_tuned)) print('Tuning time (approx): ' + str(diff / 3) + ' seconds') # we tuned 3 times # generate folds, so we know the indices of test instances at any point folds = optunity.generate_folds(data.shape[0], num_folds=3) # create another cross-validation decorator # we will compare nested cross-validation results for both tuned and untuned models # to do this, we will perform nested cross-validation but aggregate results using the identity function # this will yield the predictions outer_cv = optunity.cross_validated( x=data, y=targets, num_folds=3, folds=[folds], aggregator=optunity.cross_validation.identity) def svr_untuned_predictions(x_train, y_train, x_test, y_test): model = sklearn.svm.SVR().fit(x_train, y_train)