ConstantKernel, WhiteKernel, PairwiseKernel, KernelOperator, Exponentiation) from sklearn.base import clone from sklearn.utils.testing import (assert_equal, assert_almost_equal, assert_not_equal, assert_array_equal, assert_array_almost_equal) X = np.random.RandomState(0).normal(0, 1, (5, 2)) Y = np.random.RandomState(0).normal(0, 1, (6, 2)) kernel_white = RBF(length_scale=2.0) + WhiteKernel(noise_level=3.0) kernels = [ RBF(length_scale=2.0), RBF(length_scale_bounds=(0.5, 2.0)), ConstantKernel(constant_value=10.0), 2.0 * RBF(length_scale=0.33, length_scale_bounds="fixed"), 2.0 * RBF(length_scale=0.5), kernel_white, 2.0 * RBF(length_scale=[0.5, 2.0]), 2.0 * Matern(length_scale=0.33, length_scale_bounds="fixed"), 2.0 * Matern(length_scale=0.5, nu=0.5), 2.0 * Matern(length_scale=1.5, nu=1.5), 2.0 * Matern(length_scale=2.5, nu=2.5), 2.0 * Matern(length_scale=[0.5, 2.0], nu=0.5), 3.0 * Matern(length_scale=[2.0, 0.5], nu=1.5), 4.0 * Matern(length_scale=[0.5, 0.5], nu=2.5), RationalQuadratic(length_scale=0.5, alpha=1.5), ExpSineSquared(length_scale=0.5, periodicity=1.5), DotProduct(sigma_0=2.0), DotProduct(sigma_0=2.0)**2 ] for metric in PAIRWISE_KERNEL_FUNCTIONS:
x_pred.columns = x.columns # オートスケーリング autoscaled_y = (y - y.mean()) / y.std() autoscaled_x = (x - x.mean()) / x.std() autoscaled_x_pred = (x_pred - x.mean()) / x.std() # データ確認 pd.concat([autoscaled_y, autoscaled_x], axis=1) autoscaled_x_pred # 5 カーネルの設定 ----------------------------------------------------------- # カーネル 11 種類 kernels = [ ConstantKernel() * DotProduct() + WhiteKernel(), ConstantKernel() * RBF() + WhiteKernel(), ConstantKernel() * RBF() + WhiteKernel() + ConstantKernel() * DotProduct(), ConstantKernel() * RBF(np.ones(x.shape[1])) + WhiteKernel(), ConstantKernel() * RBF(np.ones(x.shape[1])) + WhiteKernel() + ConstantKernel() * DotProduct(), ConstantKernel() * Matern(nu=1.5) + WhiteKernel(), ConstantKernel() * Matern(nu=1.5) + WhiteKernel() + ConstantKernel() * DotProduct(), ConstantKernel() * Matern(nu=0.5) + WhiteKernel(), ConstantKernel() * Matern(nu=0.5) + WhiteKernel() + ConstantKernel() * DotProduct(), ConstantKernel() * Matern(nu=2.5) + WhiteKernel(), ConstantKernel() * Matern(nu=2.5) + WhiteKernel() + ConstantKernel() * DotProduct() ]
def main(): if len(sys.argv) < 2: print('syntax:\t%s <csv file>' % sys.argv[0]) return co2data = pd.read_csv(sys.argv[1]) print(co2data.head()) print(co2data.isnull().any()) co2data_full = co2data.copy(deep=True) co2data = co2data.dropna() print('referencing dates from 1950\ndays as % of the year') x = np.array(co2data['Decimal Date'] - 1950) m = 40 #np.mean(x) y = np.array(co2data['Carbon Dioxide (ppm)']) x_learn = featurize(x, m) y_learn = [val for val in y] x_train, x_test, x_validate, y_train, y_test, y_validate, x_oob, y_oob = katiesplit( x_learn, y_learn) quadmod = linear_model.LinearRegression().fit(x_train, y_train) print("Accuracy (test): ", quadmod.score(x_test, y_test)) print("Accuracy (validate): ", quadmod.score(x_validate, y_validate)) np.savetxt('%s.original' % sys.argv[1], np.column_stack((x, x_learn, y_learn)), fmt='%.2f') x_learn = refeaturize(x, m) y_learn -= quadmod.predict(featurize(x, m)) x_train, x_test, x_validate, y_train, y_test, y_validate, x_oob, y_oob = katiesplit( x_learn, y_learn) pertmod = linear_model.LinearRegression().fit(x_train, y_train) print("Perturbation Accuracy (test): ", pertmod.score(x_test, y_test)) print("Perturbation Accuracy (validate): ", pertmod.score(x_validate, y_validate)) pred_arg = np.linspace(0, 80, 2561) x_pred = refeaturize(pred_arg, m) y_pred = pertmod.predict(x_pred) + quadmod.predict(featurize(pred_arg, m)) np.savetxt('%s.predictions' % sys.argv[1], np.column_stack((pred_arg, x_pred, y_pred)), fmt='%.2f') ''' =============== Gaussian process section =================== ''' x_learn = x.copy() y_learn = [val for val in y] x_train, x_test, x_validate, y_train, y_test, y_validate, x_oob, y_oob = katiesplit( x_learn, y_learn) k1 = 66.0**2 * RBF(length_scale=67.0) # long term smooth rising trend k2 = 2.4**2 * RBF(length_scale=90.0) * ExpSineSquared( length_scale=1.3, periodicity=1.0) # seasonal component k3 = 0.66**2 * RationalQuadratic(length_scale=1.2, alpha=0.78) # medium term irregularity k4 = 0.18**2 * RBF(length_scale=0.134) + WhiteKernel(noise_level=0.19** 2) # noise terms kernel_gpml = k1 + k2 + k3 + k4 gp = GaussianProcessRegressor(kernel=kernel_gpml, alpha=0, optimizer=None, normalize_y=True) gp.fit( np.asarray(x_train).reshape(-1, 1), np.asarray(y_train).reshape(-1, 1)) print("GPML kernel: %s" % gp.kernel_) print("Log-marginal-likelihood: %.3f" % gp.log_marginal_likelihood(gp.kernel_.theta)) print( 'GP score (test): ', gp.score( np.asarray(x_test).reshape(-1, 1), np.asarray(y_test).reshape(-1, 1))) print( 'GP score (validate): ', gp.score( np.asarray(x_validate).reshape(-1, 1), np.asarray(y_validate).reshape(-1, 1))) # Kernel with reduced parameters #k1 = 50.0**2 * RBF(length_scale=50.0) # long term smooth rising trend k1 = 50.0**2 * RationalQuadratic( length_scale=50.0, alpha=1.0) # long term smooth rising trend #k2 = 2.0**2 * RBF(length_scale=100.0) * ExpSineSquared(length_scale=1.0, periodicity=1.0, periodicity_bounds="fixed") # seasonal component k2 = 2.0**2 * ExpSineSquared( length_scale=1.0, periodicity=1.0, periodicity_bounds=(.9, 1.1)) # seasonal component #k3 = 10**2 * RationalQuadratic(length_scale=10.0, alpha=1.0) # medium term irregularities k4 = 0.1**2 * RBF(length_scale=0.1) + WhiteKernel( noise_level=0.1**2, noise_level_bounds=(1e-3, np.inf)) # noise terms k5 = ConstantKernel(constant_value=10000, constant_value_bounds="fixed") # baseline shift kernel = k1 + k2 + k4 + k5 gp_me = GaussianProcessRegressor(kernel=kernel, alpha=0, normalize_y=True) gp_me.fit( np.asarray(x_train).reshape(-1, 1), np.asarray(y_train).reshape(-1, 1)) print("\nLearned kernel: %s" % gp_me.kernel_) print("Log-marginal-likelihood: %.3f" % gp_me.log_marginal_likelihood(gp_me.kernel_.theta)) x_pred = pred_arg[:, np.newaxis] y_pred, y_std = gp.predict(np.asarray(x_pred).reshape(-1, 1), return_std=True) y_pred_me, y_std_me = gp_me.predict(np.asarray(x_pred).reshape(-1, 1), return_std=True) print( 'GP score (test): ', gp_me.score( np.asarray(x_test).reshape(-1, 1), np.asarray(y_test).reshape(-1, 1))) print( 'GP score (validate): ', gp_me.score( np.asarray(x_validate).reshape(-1, 1), np.asarray(y_validate).reshape(-1, 1))) np.savetxt('%s.gp_predictions' % sys.argv[1], np.column_stack((x_pred, y_pred, y_std, y_pred_me, y_std_me)), fmt='%.2f') print("Now doing compound quadratic linregression, then reduced GP\n\n") print("... actually going to skip this for now") return x = np.array(co2data['Decimal Date'] - 1950) m = 40 #np.mean(x) y = np.array(co2data['Carbon Dioxide (ppm)']) x_learn = x.copy() y_learn = [val for val in y] x_train, x_test, x_validate, y_train, y_test, y_validate, x_oob, y_oob = katiesplit( x_learn, y_learn) quadmod = linear_model.LinearRegression().fit(featurize(x_oob, m), y_oob) k1 = 2.0**2 * RBF(length_scale=100.0) * ExpSineSquared( length_scale=1.0, periodicity=1.0, periodicity_bounds="fixed") # seasonal component k2 = 0.5**2 * RationalQuadratic(length_scale=1.0, alpha=1.0) # medium term irregularities k3 = 0.1**2 * RBF(length_scale=0.1) + WhiteKernel( noise_level=0.1**2, noise_level_bounds=(1e-3, np.inf)) # noise terms kernel_compound = k1 + k2 + k3 gp = GaussianProcessRegressor(kernel=kernel_compound, alpha=0, normalize_y=True) gp.fit( np.asarray(x_train).reshape(-1, 1), np.asarray(y_train - quadmod.predict(featurize(x_train, m)).reshape(-1, 1))) print("\nLearned kernel: %s" % gp.kernel_) print("Log-marginal-likelihood: %.3f" % gp.log_marginal_likelihood(gp.kernel_.theta)) y_pred = gp.predict(np.asarray(x_pred).reshape(-1, 1), return_std=False) y_qm = quadmod.predict(featurize(x_pred, m)) y_pred = np.asarray(y_pred).reshape(-1, 1) + y_qm #print('GP score (test): ', gp.score(np.asarray(x_test).reshape(-1,1), np.asarray(y_test).reshape(-1,1) - quadmod.predict(featurize(x_test,m)) )) #print('GP score (validate): ', gp.score(np.asarray(x_validate).reshape(-1,1), np.asarray(y_validate - quadmod.predict(featurize(x_validate,m))).reshape(-1,1) )) np.savetxt('%s.quad_gp_predictions' % sys.argv[1], np.column_stack((x_pred, y_pred)), fmt='%.2f') return
vec = CountVectorizer() X = vec.fit_transform(sample) pd.DataFrame(X.toarray(), columns=vec.get_feature_names()) from sklearn.feature_extraction.text import TfidfVectorizer vec = TfidfVectorizer() X = vec.fit_transform(sample) pd.DataFrame(X.toarray(), columns=vec.get_feature_names()) from sklearn.model_selection import train_test_split Xtrain, Xtest, ytrain, ytest = train_test_split(X, alphas, random_state=1) from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import ConstantKernel, RBF rbf = ConstantKernel(1.0) * RBF(length_scale=1.0) gpr = GaussianProcessRegressor(kernel=rbf, alpha=1e-8) gpr.fit(Xtrain.toarray(), ytrain) mu_s, cov_s = gpr.predict(Xtest.toarray(), return_cov=True) print("Tfid vectorizer results: ", np.corrcoef(ytest, mu_s)) # r = 0.683 bgrmvector = CountVectorizer(ngram_range=(2, 2), token_pattern=r'\b\w+\b', min_df=1) X = bgrmvector.fit_transform(sample) pd.DataFrame(X.toarray(), columns=bgrmvector.get_feature_names()) Xtrain, Xtest, ytrain, ytest = train_test_split(X, alphas, random_state=1) rbf = ConstantKernel(1.0) * RBF(length_scale=1.0) gpr = GaussianProcessRegressor(kernel=rbf, alpha=1e-8) gpr.fit(Xtrain.toarray(), ytrain) mu_s, cov_s = gpr.predict(Xtest.toarray(), return_cov=True)
'k', linewidth=2) plt.xlabel('Ratio of the number of X-variables') plt.ylabel('RMSE of OOB') plt.show() optimal_random_forest_x_variables_rate = random_forest_x_variables_rates[ np.where(rmse_oob_all == np.min(rmse_oob_all))[0][0]] regression_model = RandomForestRegressor( n_estimators=random_forest_number_of_trees, max_features=int( max( math.ceil(autoscaled_x_train.shape[1] * optimal_random_forest_x_variables_rate), 1)), oob_score=True) elif method == 'gp': # Gaussian process regression_model = GaussianProcessRegressor(ConstantKernel() * RBF() + WhiteKernel(), alpha=0) elif method == 'lgb': # LightGBM import lightgbm as lgb regression_model = lgb.LGBMRegressor() elif method == 'xgb': # XGBoost import xgboost as xgb regression_model = xgb.XGBRegressor() elif method == 'gbdt': # scikit-learn from sklearn.ensemble import GradientBoostingRegressor regression_model = GradientBoostingRegressor() regression_model.fit(autoscaled_x_train, autoscaled_y_train)
[1e-3, 1e-2, 0.1, 0.5, 0.9, 2] }), (RadiusNeighborsClassifier, { 'radius': neighbor_radius, 'algo': neighbor_algo, 'leaf_size': neighbor_leaf_size, 'metric': neighbor_metric, 'weights': ['uniform', 'distance'], 'p': [1, 2], 'outlier_label': [-1] })] gaussianprocess_models_n_params = [(GaussianProcessClassifier, { 'warm_start': warm_start, 'kernel': [RBF(), ConstantKernel(), DotProduct(), WhiteKernel()], 'max_iter_predict': [500], 'n_restarts_optimizer': [3], })] bayes_models_n_params = [(GaussianNB, {})] nn_models_n_params = [(MLPClassifier, { 'hidden_layer_sizes': [(16, ), (64, ), (100, ), (32, 32)], 'activation': ['identity', 'logistic', 'tanh', 'relu'], 'alpha': alpha, 'learning_rate': learning_rate,
axs[1].set_title("Samples from posterior distribution") fig.suptitle("Periodic kernel", fontsize=18) plt.tight_layout() # %% print(f"Kernel parameters before fit:\n{kernel})") print(f"Kernel parameters after fit: \n{gpr.kernel_} \n" f"Log-likelihood: {gpr.log_marginal_likelihood(gpr.kernel_.theta):.3f}") # %% # Dot product kernel # .................. from sklearn.gaussian_process.kernels import ConstantKernel, DotProduct kernel = ConstantKernel(0.1, (0.01, 10.0)) * (DotProduct( sigma_0=1.0, sigma_0_bounds=(0.1, 10.0))**2) gpr = GaussianProcessRegressor(kernel=kernel, random_state=0) fig, axs = plt.subplots(nrows=2, sharex=True, sharey=True, figsize=(10, 8)) # plot prior plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[0]) axs[0].set_title("Samples from prior distribution") # plot posterior gpr.fit(X_train, y_train) plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[1]) axs[1].scatter(X_train[:, 0], y_train, color="red",
from matplotlib import pyplot as plt from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import (RBF, Matern, RationalQuadratic, ExpSineSquared, DotProduct, ConstantKernel) kernels = [ 1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)), 1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1), 1.0 * ExpSineSquared(length_scale=1.0, periodicity=3.0, length_scale_bounds=(0.1, 10.0), periodicity_bounds=(1.0, 10.0)), ConstantKernel(0.1, (0.01, 10.0)) * (DotProduct(sigma_0=1.0, sigma_0_bounds=(0.1, 10.0))**2), 1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), nu=1.5) ] for fig_index, kernel in enumerate(kernels): # Specify Gaussian Process gp = GaussianProcessRegressor(kernel=kernel) # Plot prior plt.figure(fig_index, figsize=(8, 8)) plt.subplot(2, 1, 1) X_ = np.linspace(0, 5, 100) y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True) plt.plot(X_, y_mean, 'k', lw=3, zorder=9) plt.fill_between(X_, y_mean - y_std, y_mean + y_std, alpha=0.2, color='k')
y_train = sc_y.transform(y_train) # transform test dataset y_test = sc_y.transform(y_test) print('Training Features Shape:', x_train.shape) print('Training Labels Shape:', y_train.shape) print('Testing Features Shape:', x_test.shape) print('Testing Labels Shape:', y_test.shape) hyper_params = [{ 'alpha': (1e-10, ), #'alpha': (1e-2, 1e-1, 1e0, 1e1, 1e2,), #'n_restarts_optimizer': (0,1,10,100,), 'n_restarts_optimizer': (0, ), #'kernel': (DotProduct() + WhiteKernel(),), 'kernel': (ConstantKernel(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)), ), }] # 'kernel': ([1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)), # 1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1), # 1.0 * ExpSineSquared(length_scale=1.0, periodicity=3.0, # length_scale_bounds=(0.1, 10.0), # periodicity_bounds=(1.0, 10.0)), # ConstantKernel(0.1, (0.01, 10.0)) # * (DotProduct(sigma_0=1.0, sigma_0_bounds=(0.1, 10.0)) ** 2), # 1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), nu=1.5)],), }] # "kernel": ([ExpSineSquared(l, p) # for l in np.logspace(-2, 2, 10) # for p in np.logspace(0, 2, 10)], # [DotProduct() + WhiteKernel()],
def add_parameter_ui(classifier_name): param = dict() if classifier_name == "KNN": k = st.sidebar.slider("K", 1, 15) param["K"] = k elif classifier_name == "SVM": C = st.sidebar.slider("C", 0.01, 10.0) param["C"] = C elif classifier_name == "Random Forest": max_depth = st.sidebar.slider("Max depth", 2, 16) estimator = st.sidebar.slider("estimator", 1, 100) param["max_depth"] = max_depth param["estimator"] = estimator elif classifier_name == "BaggingClassifier": param["base_estimator"] = SVC() n_estimator = st.sidebar.slider("n_estimator", 1, 20) max_sample = st.sidebar.slider("Max Sample", 0.1, 1.0) param["n_estimator"] = n_estimator param["max_sample"] = max_sample elif classifier_name == "DecisionTreeClassifier": max_depth = st.sidebar.slider("Max depth", 2, 16) leaf = st.sidebar.slider("Leaf Nodes", 1, 100) param["max_depth"] = max_depth param["Leaf"] = leaf elif classifier_name == "ExtraTreeClassifier": max_depth = st.sidebar.slider("Max depth", 2, 16) leaf = st.sidebar.slider("Leaf Nodes", 1, 100) min_sample_split = st.sidebar.slider("min_sample_split", 0.1, 5.0) param["max_depth"] = max_depth param["Leaf"] = leaf if min_sample_split <= 1: param["min_sample_split"] = float(min_sample_split) elif min_sample_split > 1 and min_sample_split < 2: param["min_sample_split"] = int(min_sample_split) + 1 else: param["min_sample_split"] = int(min_sample_split) elif classifier_name == "GaussianProcessClassifier": n_restarts_optimizer = st.sidebar.slider("n_restarts_optimizer", 0, 10) max_iter_predict = st.sidebar.slider("max_iter_predict", 50, 100) l = [ 1 * RBF(), 1 * DotProduct(), 1 * ConstantKernel(), 1 * RationalQuadratic() ] kernal = st.sidebar.selectbox("Kernal", l) param["optimizer"] = n_restarts_optimizer param["predict"] = max_iter_predict param["kernal"] = kernal elif classifier_name == "LinearSVC": loss = st.sidebar.selectbox("loss", ('hinge', 'squared_hinge')) C = st.sidebar.slider("C", 0.1, 5.0) param["loss"] = loss param["c"] = C elif classifier_name == "NuSVC": kernel = st.sidebar.selectbox( "kernel", ('linear', 'poly', 'rbf', 'sigmoid', 'precomputed')) degree = st.sidebar.slider("degree", 0, 5) gamma = st.sidebar.selectbox("Gamma", ('scale', 'auto')) param["kernel"] = kernel param["degree"] = degree param["gamma"] = gamma elif classifier_name == "AdaBoostClassifier": n_estimator = st.sidebar.slider("n_estimators", 10, 100) learning_rate = st.sidebar.slider("Learning Rate", 0.01, 1.00) algorithm = st.sidebar.selectbox("Algorithm", ('SAMME', 'SAMME.R')) param["estimators"] = n_estimator param["learning"] = learning_rate param["algo"] = algorithm return param
# strategy strategy = SRBFStrategy(lb, ub) # uncertainty information mu = 0.0 sigma = np.sqrt(0.01) p = lambda num_pts: np.random.normal(mu, sigma, (num_pts, dim)) # Monte Carlo parameters num_points_MC = 5000 # Mean Variance weighting eta = 0.5 # surrogate kernel = ConstantKernel(1, (1e-3, 1e3)) * RBF(1, (0.1, 100)) + \ WhiteKernel(1e-3, (1e-6, 1e-2)) surrogate = MeanVariance(kernel, p, eta, num_points_MC) # initialize the problem problem = BayesianOptimization(f, dim, max_evals, exp_design, strategy, surrogate, lb, ub) # solve it xopt, fopt = problem.minimize() #============================================================= # Plot #============================================================= # test points Ntest = 300
def special_conversions(self, classifier_params): """ TODO: Make this logic into subclasses ORRRR, should make each enumerator handle it in a static function something like: @staticmethod def ParamsTransformation(params_dict): # does classifier-specific changes return params_dict """ # do special converstions ### RF ### if "n_jobs" in classifier_params: classifier_params["n_jobs"] = int(classifier_params["n_jobs"]) if "n_estimators" in classifier_params: classifier_params["n_estimators"] = int( classifier_params["n_estimators"]) ### PCA ### if "_pca" in classifier_params: del classifier_params["_pca"] del classifier_params["_pca_dimensions"] ### GPC ### if classifier_params["function"] == "gp": if classifier_params["kernel"] == "constant": classifier_params["kernel"] = ConstantKernel() elif classifier_params["kernel"] == "rbf": classifier_params["kernel"] = RBF() elif classifier_params["kernel"] == "matern": classifier_params["kernel"] = Matern( nu=classifier_params["nu"]) del classifier_params["nu"] elif classifier_params["kernel"] == "rational_quadratic": classifier_params["kernel"] = RationalQuadratic( length_scale=classifier_params["length_scale"], alpha=classifier_params["alpha"]) del classifier_params["length_scale"] del classifier_params["alpha"] elif classifier_params["kernel"] == "exp_sine_squared": classifier_params["kernel"] = ExpSineSquared( length_scale=classifier_params["length_scale"], periodicity=classifier_params["periodicity"]) del classifier_params["length_scale"] del classifier_params["periodicity"] ### MLP ### if classifier_params["function"] == "mlp": classifier_params["hidden_layer_sizes"] = [] # set layer topology if int(classifier_params["num_hidden_layers"]) == 1: classifier_params["hidden_layer_sizes"].append( classifier_params["hidden_size_layer1"]) del classifier_params["hidden_size_layer1"] elif int(classifier_params["num_hidden_layers"]) == 2: classifier_params["hidden_layer_sizes"].append( classifier_params["hidden_size_layer1"]) classifier_params["hidden_layer_sizes"].append( classifier_params["hidden_size_layer2"]) del classifier_params["hidden_size_layer1"] del classifier_params["hidden_size_layer2"] elif int(classifier_params["num_hidden_layers"]) == 3: classifier_params["hidden_layer_sizes"].append( classifier_params["hidden_size_layer1"]) classifier_params["hidden_layer_sizes"].append( classifier_params["hidden_size_layer2"]) classifier_params["hidden_layer_sizes"].append( classifier_params["hidden_size_layer3"]) del classifier_params["hidden_size_layer1"] del classifier_params["hidden_size_layer2"] del classifier_params["hidden_size_layer3"] classifier_params["hidden_layer_sizes"] = [ int(x) for x in classifier_params["hidden_layer_sizes"] ] # convert to ints # delete our fabricated keys del classifier_params["num_hidden_layers"] # print "Added stuff for DBNs! %s" % classifier_params ### DBN ### if classifier_params["function"] == "dbn": # print "Adding stuff for DBNs! %s" % classifier_params classifier_params["layer_sizes"] = [ classifier_params["inlayer_size"] ] # set layer topology if int(classifier_params["num_hidden_layers"]) == 1: classifier_params["layer_sizes"].append( classifier_params["hidden_size_layer1"]) del classifier_params["hidden_size_layer1"] elif int(classifier_params["num_hidden_layers"]) == 2: classifier_params["layer_sizes"].append( classifier_params["hidden_size_layer1"]) classifier_params["layer_sizes"].append( classifier_params["hidden_size_layer2"]) del classifier_params["hidden_size_layer1"] del classifier_params["hidden_size_layer2"] elif int(classifier_params["num_hidden_layers"]) == 3: classifier_params["layer_sizes"].append( classifier_params["hidden_size_layer1"]) classifier_params["layer_sizes"].append( classifier_params["hidden_size_layer2"]) classifier_params["layer_sizes"].append( classifier_params["hidden_size_layer3"]) del classifier_params["hidden_size_layer1"] del classifier_params["hidden_size_layer2"] del classifier_params["hidden_size_layer3"] classifier_params["layer_sizes"].append( classifier_params["outlayer_size"]) classifier_params["layer_sizes"] = [ int(x) for x in classifier_params["layer_sizes"] ] # convert to ints # set activation function if classifier_params["output_act_funct"] == "Linear": classifier_params["output_act_funct"] = Linear() elif classifier_params["output_act_funct"] == "Sigmoid": classifier_params["output_act_funct"] = Sigmoid() elif classifier_params["output_act_funct"] == "Softmax": classifier_params["output_act_funct"] = Softmax() elif classifier_params["output_act_funct"] == "tanh": classifier_params["output_act_funct"] = Tanh() classifier_params["epochs"] = int(classifier_params["epochs"]) # delete our fabricated keys del classifier_params["num_hidden_layers"] del classifier_params["inlayer_size"] del classifier_params["outlayer_size"] # remove function key and return del classifier_params["function"] return classifier_params
for x0 in np.random.uniform(bounds[:, 0], bounds[:, 1], size=(n_restarts, dim)): res = minimize(min_obj, x0=x0, bounds=bounds, method='L-BFGS-B') if res.fun < min_val: min_val = res.fun[0] min_x = res.x return min_x.reshape(-1, 1) from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import ConstantKernel, Matern # Gaussian process with Mat??rn kernel as surrogate model m52 = ConstantKernel(1.0) * Matern(length_scale=1.0, nu=2.5) gpr = GaussianProcessRegressor(kernel=m52, alpha=noise**2) # Initialize samples X_sample = X_init Y_sample = Y_init # Number of iterations n_iter = 30 plt.figure(figsize=(12, n_iter * 3)) plt.subplots_adjust(hspace=0.4) for i in range(n_iter): # Update Gaussian process with existing samples gpr.fit(X_sample, Y_sample)
n_train = 3000 n_test = 10000 opt = Adam(1e-3) batch_size = 500 gp = NNRegressor(layers, opt=opt, batch_size=batch_size, maxiter=500, gp=True, verbose=True) gp.fit(x_train, y_train) #Can extract mapping z(x) and hyperparams for use in other learning algorithm alph = gp.layers[-1].s_alpha var = gp.layers[-1].var A_full = gp.fast_forward(x_train) kernel = ConstantKernel(var) * RBF(np.ones(1)) + WhiteKernel(alph) A_test = gp.fast_forward(x_test[0:n_test]) yp = np.zeros(n_test) std = np.zeros(n_test) gp1 = GaussianProcessRegressor(kernel, optimizer=None) gp1.fit(A_full[0:500], y_train[0:500]) mu, stdt = gp1.predict(A_test, return_std=True) print("GP Regression:") print(np.sqrt(np.mean((np.rint(mu) - y_test)**2)))
def interpolationLearner(xtrain, ytrain, xtest): estimators = [] #model1 param_grid = {"alpha": [5e-05], "tol": [0.0001, 0.00001]} model1 = GridSearchCV(Lasso(), cv=4, param_grid=param_grid, scoring="neg_mean_squared_error") estimators.append( ('Lasso', model1)) # https://www.kaggle.com/apapiu/regularized-linear-models #model2 param_grid = { "alpha": [5e-05, 0.0005], "l1_ratio": [0.9, 0.8], "random_state": [3, 4] } model2 = GridSearchCV( ElasticNet(), cv=4, param_grid=param_grid, scoring="neg_mean_squared_error" ) # http://scikit-learn.org/stable/auto_examples/gaussian_process/plot_compare_gpr_krr.html#sphx-glr-auto-examples-gaussian-process-plot-compare-gpr-krr-py estimators.append(('ENet', model2)) #model3 param_grid = { "alpha": [0.6, 0.5], "kernel": [ConstantKernel(1.0, (1e-3, 1e3)) + RBF()] } model3 = GridSearchCV( KernelRidge(), cv=4, param_grid=param_grid, scoring="neg_mean_squared_error" ) # http://scikit-learn.org/stable/auto_examples/gaussian_process/plot_compare_gpr_krr.html#sphx-glr-auto-examples-gaussian-process-plot-compare-gpr-krr-py estimators.append(('KRR', model3)) #model4 param_grid = { "loss": ["huber"], "max_features": ["sqrt"], "max_depth": [4, 5, 6] } model4 = GridSearchCV( GradientBoostingRegressor(learning_rate=0.05, max_depth=3, n_estimators=2200), cv=4, param_grid=param_grid, scoring="neg_mean_squared_error" ) # http://scikit-learn.org/stable/auto_examples/gaussian_process/plot_compare_gpr_krr.html#sphx-glr-auto-examples-gaussian-process-plot-compare-gpr-krr-py estimators.append(('GBoost', model4)) learners = [] # Hagamos fit for learner in np.arange(len(estimators)): stime = time.time() learners.append(estimators[learner][1].fit(xtrain, ytrain)) print("For " + str(estimators[learner][0]).upper()) print("") print("Time for learner " + str(estimators[learner][0]) + " " + str(time.time() - stime)) print("El learner " + str(estimators[learner][0]) + " tiene un score de " + str(learners[learner].best_score_)) print("Los parametros para el learner " + str(estimators[learner][0]) + " fueron " + str(learners[learner].best_params_)) print("") joblib.dump(model1, 'model_cache/int_lasso_cache.pkl') joblib.dump(model2, 'model_cache/int_elastic_cache.pkl') joblib.dump(model3, 'model_cache/int_kernel_cache.pkl') joblib.dump(model4, 'model_cache/int_gboost_cache.pkl') # CV score para la media de los learners predictTraino = np.column_stack( [learner.predict(xtrain) for learner in learners]) mean_predictTraino = np.column_stack([np.mean(i) for i in predictTraino]) print("") print("El score final de toda la interpolación en traino es " + str(mean_squared_error(ytrain, mean_predictTraino[0]))) print("") predict = np.column_stack([learner.predict(xtest) for learner in learners]) mean_predict = np.column_stack([i.mean() for i in predict]) return mean_predict[0]
def __init__(self): super(Selector, self).__init__() kernel = ConstantKernel(1.0) * Matern(length_scale=1.0, nu=2.5) # self.model = GaussianProcessRegressor(alpha=0.2 ** 2) self.model = GaussianProcessRegressor(kernel, alpha=0.2**2) self.name = "GP_opt"
def cross_validation(self, X, y, cv = 5, scoring = 'neg_mean_squared_error', scale = True, fit = True): """You can decide kernels by using cross validation. You don't have to do this if you've already decide which kernels do you use. Parameters ---------- X : list or something [description] y : list or something [description] cv : int, optional [description], by default 5 scoring : str, optional [description], by default 'neg_mean_squared_error' scale : bool, optional [description], by default True fit : bool, optional [description], by default True """ # クラス変数化 self.X = np.array(X).reshape(-1, 1) self.y = np.array(y).reshape(-1, 1) cv = min(cv, self.y.shape[0]) # 目的変数を正規化 if scale: scaled_y, self.scaler_y = self._scale(self.y) else: scaled_y = self.y self.scaler_y = None # カーネル kernels = [ConstantKernel() * DotProduct() + WhiteKernel(), ConstantKernel() * RBF() + WhiteKernel(), ConstantKernel() * RBF() + WhiteKernel() + ConstantKernel() * DotProduct(), ConstantKernel() * RBF(np.ones(self.X.shape[1])) + WhiteKernel(), ConstantKernel() * RBF(np.ones(self.X.shape[1])) + WhiteKernel() + ConstantKernel() * DotProduct(), ConstantKernel() * Matern(nu=1.5) + WhiteKernel(), ConstantKernel() * Matern(nu=1.5) + WhiteKernel() + ConstantKernel() * DotProduct(), ConstantKernel() * Matern(nu=0.5) + WhiteKernel(), ConstantKernel() * Matern(nu=0.5) + WhiteKernel() + ConstantKernel() * DotProduct(), ConstantKernel() * Matern(nu=2.5) + WhiteKernel(), ConstantKernel() * Matern(nu=2.5) + WhiteKernel() + ConstantKernel() * DotProduct() ] params = { 'kernel':kernels } # GPRのインスタンス生成 gpr = GaussianProcessRegressor(alpha=0) # kernel決定のためにcv gscv = GridSearchCV(gpr, params, cv = cv, scoring = scoring) gscv.fit(self.X, scaled_y) # 後でアクセスできるように self.best_score_ = gscv.best_score_ self.results = pd.DataFrame.from_dict(gscv.cv_results_) # 最適なカーネルを使用 self.best_kernel_ = gscv.best_params_['kernel'] self.best_estimator = GaussianProcessRegressor(kernel = self.best_kernel_, alpha = 0) if fit: # fitさせる self.best_estimator.fit(self.X, scaled_y)
max_sigma = [0] * total_increments #* max_n max_error = [0] * total_increments #* max_n print("bruh") for index in range(1, len( x_axis_array)): # max_n + 1): # i represents the current value of n i = x_axis_array[index] print("Finding every " + str(i) + "th element") start_time = time.time() X_train = normalize(X_train_original[::i]) X_test = normalize(X_test_original[::i]) y_train = normalize(y_train_original[::i]) y_test = normalize(y_test_original[::i]) kernel = ConstantKernel(1.0, (1e-3, 1e3)) * RBF([5] * X_train.shape[1], (1e-2, 1e2)) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=15) gp.fit(X_train, y_train) y_pred, stdev = gp.predict(X_test, return_std=True) end_time = time.time() print("done training") time_array[index] = end_time - start_time num_data_points[index] = len(y_train) #maximum sigma max_sigma[index] = max(stdev) #maximum relative error relative_error = (y_pred - y_test) / (y_test) max_error[index] = max(relative_error)
def bayesianoptimization(X, y, candidates_of_X, acquisition_function_flag, cumulative_variance=None): """ Bayesian optimization Gaussian process regression model is constructed between X and y. A candidate of X with the highest acquisition function is selected using the model from candidates of X. Parameters ---------- X: numpy.array or pandas.DataFrame m x n matrix of X-variables of training dataset (m is the number of samples and n is the number of X-variables) y: numpy.array or pandas.DataFrame m x 1 vector of a y-variable of training dataset candidates_of_X: numpy.array or pandas.DataFrame Candidates of X acquisition_function_flag: int 1: Mutual information (MI), 2: Expected improvement(EI), 3: Probability of improvement (PI) [0: Estimated y-values] cumulative_variance: numpy.array or pandas.DataFrame cumulative variance in mutual information (MI)[acquisition_function_flag=1] Returns ------- selected_candidate_number : int selected number of candidates_of_X selected_X_candidate : numpy.array selected X candidate cumulative_variance: numpy.array cumulative variance in mutual information (MI)[acquisition_function_flag=1] """ X = np.array(X) y = np.array(y) if cumulative_variance is None: cumulative_variance = np.empty(len(y)) else: cumulative_variance = np.array(cumulative_variance) relaxation_value = 0.01 delta = 10**-6 alpha = np.log(2 / delta) autoscaled_X = (X - X.mean(axis=0)) / X.std(axis=0, ddof=1) autoscaled_candidates_of_X = (candidates_of_X - X.mean(axis=0)) / X.std( axis=0, ddof=1) autoscaled_y = (y - y.mean(axis=0)) / y.std(axis=0, ddof=1) gaussian_process_model = GaussianProcessRegressor( ConstantKernel() * RBF() + WhiteKernel(), alpha=0) gaussian_process_model.fit(autoscaled_X, autoscaled_y) autoscaled_estimated_y_test, autoscaled_std_of_estimated_y_test = gaussian_process_model.predict( autoscaled_candidates_of_X, return_std=True) if acquisition_function_flag == 1: acquisition_function_values = autoscaled_estimated_y_test + alpha**0.5 * ( (autoscaled_std_of_estimated_y_test**2 + cumulative_variance)**0.5 - cumulative_variance**0.5) cumulative_variance = cumulative_variance + autoscaled_std_of_estimated_y_test**2 elif acquisition_function_flag == 2: acquisition_function_values = (autoscaled_estimated_y_test - max(autoscaled_y) - relaxation_value) * \ norm.cdf((autoscaled_estimated_y_test - max(autoscaled_y) - relaxation_value) / autoscaled_std_of_estimated_y_test) + \ autoscaled_std_of_estimated_y_test * \ norm.pdf((autoscaled_estimated_y_test - max(autoscaled_y) - relaxation_value) / autoscaled_std_of_estimated_y_test) elif acquisition_function_flag == 3: acquisition_function_values = norm.cdf( (autoscaled_estimated_y_test - max(autoscaled_y) - relaxation_value) / autoscaled_std_of_estimated_y_test) elif acquisition_function_flag == 0: acquisition_function_values = autoscaled_estimated_y_test selected_candidate_number = np.where( acquisition_function_values == max(acquisition_function_values))[0][0] selected_X_candidate = candidates_of_X[selected_candidate_number, :] return selected_candidate_number, selected_X_candidate, cumulative_variance
def make_plot(days_ago, dates, mag): print('Making plot...') time_span = np.max(dates) - np.min(dates) min_plot = 0.0 max_plot = 2 x_days = -120 # Make daily bins nights = np.arange(0, 120, 1) daily_mags = [] errors = [] for night in nights: selector = np.where((days_ago < night + 1) & (days_ago > night)) n_obs = np.size(mag[selector]) flux = biweight_location(mag[selector]) error = np.std(mag[selector]) / np.sqrt(n_obs) if error > 0.75: error = 0 daily_mags.append(flux) errors.append(error) print(night, flux, error, n_obs, np.std(mag[selector])) nights_all = nights.copy() daily_mags_all = daily_mags.copy() errors_all = errors.copy() lookback = np.arange(1, 20, 1) for missing_days in lookback: nights = nights_all.copy()[missing_days:] daily_mags = daily_mags_all.copy()[missing_days:] errors = errors_all.copy()[missing_days:] plt.errorbar(-(nights + 0.5), daily_mags, yerr=errors, fmt='.k', alpha=0.5) plt.xlabel('Days from today') plt.ylabel('Visual magnitude') mid = biweight_location(mag) plt.ylim(min_plot, max_plot) plt.xlim(-100, 100) plt.gca().invert_yaxis() date_text = datetime.datetime.now().strftime("%d %b %Y") plt.text(95, min_plot + 0.1, 'AAVSO visual (by-eye) daily bins', ha='right') plt.text(95, min_plot + 0.2, 'Gaussian process regression, Matern 3/2 kernel', ha='right') plt.text(95, min_plot + 0.3, '@betelbot update ' + date_text, ha='right') use_days = 60 - missing_days X = np.array(nights + 0.5) X = X[:use_days] y = np.array(daily_mags) y = y[:use_days] X, y = cleaned_array(X, y) length_scale = 1 kernel = ConstantKernel() + Matern( length_scale=length_scale, nu=3 / 2) + WhiteKernel(noise_level=1) X = X.reshape(-1, 1) gp = gaussian_process.GaussianProcessRegressor(kernel=kernel) gp.fit(X, y) GaussianProcessRegressor(alpha=1e-10, copy_X_train=True, kernel=1**2 + Matern(length_scale=length_scale, nu=1.5) + WhiteKernel(noise_level=1), n_restarts_optimizer=0, normalize_y=False, optimizer='fmin_l_bfgs_b', random_state=None) x_pred = np.linspace(60, -120, 250).reshape(-1, 1) y_pred, sigma = gp.predict(x_pred, return_std=True) plt.plot(-x_pred, y_pred, linestyle='dashed', color='blue') plt.fill_between(-x_pred.ravel(), y_pred + sigma, y_pred - sigma, alpha=0.5) idx = 20 - missing_days if idx < 10: filename = "0" + str(idx) + '.png' else: filename = str(idx) + '.png' plt.savefig(filename, bbox_inches='tight', dpi=100) print('Plot made', filename) plt.clf()
def main(_): mnist = input_data.read_data_sets('/tmp/data/', one_hot=True, seed=12345) random_weight_vector = np.random.uniform(low=0.1, high=1.9, size=TRAIN_INPUT_SIZE) x = tf.placeholder(tf.float32, shape=(None, INPUT_DIM), name='x') y = tf.placeholder(tf.float32, shape=(None, OUTPUT_DIM), name='y') weight = tf.placeholder(tf.float32, shape=(None, OUTPUT_DIM), name='weight') parallel_alphas = tf.placeholder(tf.float32, shape=(FLAGS.num_parallel_alphas, OUTPUT_DIM), name='parallel_alphas') unstack_parallel_alphas = tf.unstack(parallel_alphas, axis=0) parallel_logits = [] parallel_losses = [] parallel_optimizers = [] validation_metrics = [] test_metrics = [] all_test_metrics = [] with tf.variable_scope('classifier'): for alpha_index in range(FLAGS.num_parallel_alphas): logits = classifier(x) alpha = tf.reshape(unstack_parallel_alphas[alpha_index], shape=[OUTPUT_DIM, 1]) optimizer, loss = optimization(logits, y, weight, alpha, LEARNING_RATE) parallel_logits.append(logits) parallel_losses.append(loss) parallel_optimizers.append(optimizer) init = tf.global_variables_initializer() classifiers_init = tf.variables_initializer( tf.global_variables(scope='classifier')) with tf.Session() as sess: sess.run(init) # GetCandidatesAlpha (Algorithm 2 in paper) sample_alphas = np.zeros(shape=(0, OUTPUT_DIM)) for alpha_batch_index in range(FLAGS.num_alpha_batches): sess.run(classifiers_init) if FLAGS.uniform_weights: alpha_batch = np.zeros(shape=(FLAGS.num_parallel_alphas, OUTPUT_DIM)) elif FLAGS.random_alpha or alpha_batch_index < 1: alpha_batch = sample_from_ball( size=(FLAGS.num_parallel_alphas, OUTPUT_DIM), sampling_radius=FLAGS.sampling_radius) sample_alphas = np.concatenate([sample_alphas, alpha_batch]) else: # Use LCB to generate candidates. alpha_batch = np.zeros(shape=(0, OUTPUT_DIM)) sample_metrics = validation_metrics[:] for alpha_index in range(FLAGS.num_parallel_alphas): kernel = RBF(length_scale=FLAGS.sampling_radius, length_scale_bounds=( FLAGS.sampling_radius * 1e-3, FLAGS.sampling_radius * 1e3)) * ConstantKernel(1.0, (1e-3, 1e3)) gp = GaussianProcessRegressor(kernel=kernel, alpha=1e-4).fit( sample_alphas, np.log1p(sample_metrics)) candidates = sample_from_ball((10000, OUTPUT_DIM), FLAGS.sampling_radius) metric_mles, metric_stds = gp.predict(candidates, return_std=True) metric_lcbs = np.maximum( np.expm1(metric_mles - 1.0 * metric_stds), 0.0) metric_lcbs += np.random.random( size=metric_lcbs.shape) * 0.001 # break ties best_index = np.argmin(metric_lcbs) best_alpha = [candidates[best_index]] best_alpha_metric_estimate = np.minimum( np.expm1(metric_mles[best_index] + 1.0 * metric_stds[best_index]), 1.0) alpha_batch = np.concatenate([alpha_batch, best_alpha]) sample_alphas = np.concatenate([sample_alphas, best_alpha]) sample_metrics.append(best_alpha_metric_estimate) # Training classifiers for step in range(TRAINING_STEPS): batch_index = range( step * BATCH_SIZE % TRAIN_INPUT_SIZE, step * BATCH_SIZE % TRAIN_INPUT_SIZE + BATCH_SIZE) (batch_x, batch_y) = mnist.train.next_batch(BATCH_SIZE, shuffle=False) batch_weight = [[random_weight_vector[i]] * OUTPUT_DIM for i in batch_index] _, _ = sess.run( [parallel_optimizers, parallel_losses], feed_dict={ x: batch_x, y: batch_y, weight: batch_weight, parallel_alphas: alpha_batch, }) parallel_validation_logits = sess.run(parallel_logits, feed_dict={ x: mnist.validation.images, y: mnist.validation.labels, }) parallel_validation_metrics = [ metric(mnist.validation.labels, validation_logits, all_digits=False) for validation_logits in parallel_validation_logits ] validation_metrics.extend(parallel_validation_metrics) parallel_test_logits = sess.run(parallel_logits, feed_dict={ x: mnist.test.images, y: mnist.test.labels, }) parallel_test_metrics = [ metric(mnist.test.labels, test_logits, all_digits=False) for test_logits in parallel_test_logits ] test_metrics.extend(parallel_test_metrics) parallel_all_test_metrics = [ metric(mnist.test.labels, test_logits, all_digits=True) for test_logits in parallel_test_logits ] all_test_metrics.extend(parallel_all_test_metrics) best_observed_index = np.argmin(validation_metrics) print('[metric] validation={}'.format( validation_metrics[best_observed_index])) print('[metric] test={}'.format(test_metrics[best_observed_index])) for i in range(10): print('[all test metrics] {}={}'.format( i, all_test_metrics[best_observed_index][i]))
def run_model(X,Y,perc,nbre_fois,model_name,list_variable): regRandomForestRegressor_m1,regRandomForestRegressor_m2,regRandomForestRegressor_m3,regRandomForestRegressor_m4,regRandomForestRegressor_m5=None,None,None,None,None "ensemble des modeles crées" memoire_score=dict() if 'RandomForestRegressor_m' in model_name: regRandomForestRegressor_m1=RandomForestRegressor(max_depth=2, random_state=0) regRandomForestRegressor_m2=RandomForestRegressor(max_depth=7, random_state=0) regRandomForestRegressor_m3=RandomForestRegressor(max_depth=12, random_state=0) regRandomForestRegressor_m4=RandomForestRegressor(max_depth=14, random_state=0) regRandomForestRegressor_m5=RandomForestRegressor(max_depth=18, random_state=0) if 'treeDecision_regresion_m' in model_name: regtreeDecision_regresion_m1=tree.DecisionTreeRegressor(max_depth=2) regtreeDecision_regresion_m2=tree.DecisionTreeRegressor(max_depth=7) regtreeDecision_regresion_m3=tree.DecisionTreeRegressor(max_depth=12) regtreeDecision_regresion_m4=tree.DecisionTreeRegressor(max_depth=14) regtreeDecision_regresion_m5=tree.DecisionTreeRegressor(max_depth=18) if 'svm_m' in model_name: regvm_m1=svm.SVR() regvm_m2=svm.LinearSVR() if 'linear_model_OLS' in model_name: reglinearie = linear_model.LinearRegression() if 'KNeighborsRegressor_m' in model_name: regKNeighborsRegressor=KNeighborsRegressor(n_neighbors=2) if 'linear_model_Ridge' in model_name: reglinear_model_Ridge1 = linear_model.Ridge(alpha=0.1) reglinear_model_Ridge2 = linear_model.Ridge(alpha=0.1) reglinear_model_Ridge3 = linear_model.Ridge(alpha=0.1) reglinear_model_Ridge4 = linear_model.Ridge(alpha=0.1) reglinear_model_Ridge5 = linear_model.Ridge(alpha=0.5) reglinear_model_Ridge6 = linear_model.Ridge(alpha=0.6) reglinear_model_Ridge7 = linear_model.Ridge(alpha=0.7) reglinear_model_Ridge8 = linear_model.Ridge(alpha=0.8) reglinear_model_Ridge9 = linear_model.Ridge(alpha=0.9) if 'BayesianRidge_m' in model_name: regBayesianRidge_m_F=linear_model.BayesianRidge(compute_score=False) regBayesianRidge_m_T=linear_model.BayesianRidge(compute_score=True) if 'AdaBoostRegressor_m' in model_name: regAdaBoostRegressor_m=AdaBoostRegressor(random_state=0, n_estimators=100) if 'MLPregression' in model_name: reqMLPregression1=MLPRegressor(activation="identity",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression2=MLPRegressor(activation="identity",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression3=MLPRegressor(activation="identity",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression4=MLPRegressor(activation="logistic",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression5=MLPRegressor(activation="logistic",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression6=MLPRegressor(activation="logistic",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression7=MLPRegressor(activation="tanh",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression8=MLPRegressor(activation="tanh",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression9=MLPRegressor(activation="tanh",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression10=MLPRegressor(activation="relu",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression11=MLPRegressor(activation="relu",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression12=MLPRegressor(activation="relu",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression13=MLPRegressor(activation="identity",solver="sgd", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression14=MLPRegressor(activation="identity",solver="sgd", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression15=MLPRegressor(activation="identity",solver="sgd", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression16=MLPRegressor(activation="logistic",solver="sgd", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression17=MLPRegressor(activation="logistic",solver="sgd", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression18=MLPRegressor(activation="logistic",solver="sgd", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression19=MLPRegressor(activation="tanh",solver="sgd", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression20=MLPRegressor(activation="tanh",solver="sgd", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression21=MLPRegressor(activation="relu",solver="sgd", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression22=MLPRegressor(activation="relu",solver="sgd", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression23=MLPRegressor(activation="relu",solver="sgd", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression24=MLPRegressor(activation="relu",solver="sgd", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression25=MLPRegressor(activation="identity",solver="adam", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression26=MLPRegressor(activation="identity",solver="adam", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression27=MLPRegressor(activation="identity",solver="adam", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression28=MLPRegressor(activation="logistic",solver="adam", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression29=MLPRegressor(activation="logistic",solver="adam", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression30=MLPRegressor(activation="logistic",solver="adam", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression31=MLPRegressor(activation="tanh",solver="adam", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression32=MLPRegressor(activation="tanh",solver="adam", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression33=MLPRegressor(activation="tanh",solver="adam", alpha=1e-7,hidden_layer_sizes=(20,1)) reqMLPregression34=MLPRegressor(activation="relu",solver="adam", alpha=1e-7,hidden_layer_sizes=(100,10)) reqMLPregression35=MLPRegressor(activation="relu",solver="adam", alpha=1e-7,hidden_layer_sizes=(70,30)) reqMLPregression36=MLPRegressor(activation="relu",solver="adam", alpha=1e-7,hidden_layer_sizes=(20,1)) if 'GaussianProcessRegressor_m' in model_name: kernels = [1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)), 1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1), 1.0 * ExpSineSquared(length_scale=1.0, periodicity=3.0, length_scale_bounds=(0.1, 10.0), periodicity_bounds=(1.0, 10.0)), ConstantKernel(0.1, (0.01, 10.0)) * (DotProduct(sigma_0=1.0, sigma_0_bounds=(0.1, 10.0)) ** 2), 1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), nu=1.5),DotProduct() + WhiteKernel()] gpr1 = GaussianProcessRegressor(kernel=kernels[0],random_state=0) gpr2 = GaussianProcessRegressor(kernel=kernels[1],random_state=0) gpr3 = GaussianProcessRegressor(kernel=kernels[2],random_state=0) gpr4 = GaussianProcessRegressor(kernel=kernels[3],random_state=0) gpr5 = GaussianProcessRegressor(kernel=kernels[4],random_state=0) gpr6 = GaussianProcessRegressor(kernel=kernels[5],random_state=0) if 'SGDRegressor_m' in model_name: regSGDRegressor_m1=linear_model.SGDRegressor(loss="squared_loss",max_iter=1000, tol=0.001) regSGDRegressor_m2=linear_model.SGDRegressor(loss="huber",max_iter=1000, tol=0.001) regSGDRegressor_m3=linear_model.SGDRegressor(loss="epsilon_insensitive",max_iter=1000, tol=0.001) regSGDRegressor_m4=linear_model.SGDRegressor(loss="squared_epsilon_insensitive",max_iter=1000, tol=0.001) X=X[list(list_variable)] for k in range(1,nbre_fois): train_index=[] test_index=sample(list(X.index),perc) for k in list(X.index): if k not in test_index: train_index.append(k) X_test=X.iloc[test_index] # Chargement du X test y_test=Y.iloc[test_index] # Chargement du y test X_train=X.iloc[train_index] # Chargement du X train y_train=Y.iloc[train_index] # Chargement du y train if 'RandomForestRegressor_m' in model_name: # arbre de decision RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m1) RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m2) RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m3) RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m4) RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m5) if 'treeDecision_regresion_m' in model_name: treeDecision_regresion_m(X_train, y_train,X_test, y_test,regtreeDecision_regresion_m1) treeDecision_regresion_m(X_train, y_train,X_test, y_test,regtreeDecision_regresion_m2) treeDecision_regresion_m(X_train, y_train,X_test, y_test,regtreeDecision_regresion_m3) treeDecision_regresion_m(X_train, y_train,X_test, y_test,regtreeDecision_regresion_m4) treeDecision_regresion_m(X_train, y_train,X_test, y_test,regtreeDecision_regresion_m5) if 'MLPregression' in model_name: try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression1) except: print("Toto") pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression2) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression3) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression4) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression5) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression6) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression7) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression8) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression9) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression10) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression11) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression12) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression13) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression14) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression15) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression16) except: pass try: MLPregression(X_train, y_train,X_test, y_test,reqMLPregression17) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression18) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression19) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression20) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression21) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression22) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression23) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression24) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression25) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression26) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression27) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression28) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression29) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression30) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression31) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression32) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression33) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression34) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression35) MLPregression(X_train, y_train,X_test, y_test,reqMLPregression36) except: pass if 'AdaBoostRegressor_m' in model_name: try: AdaBoostRegressor_m(X_train, y_train,X_test, y_test,regAdaBoostRegressor_m) except: pass if 'KNeighborsRegressor_m' in model_name: try: KNeighborsRegressor_m(X_train, y_train,X_test, y_test,regKNeighborsRegressor) except: pass "model lineaire" if 'linear_model_OLS' in model_name: try: "Ordinary Least Squares" linear_model_OLS(X_train, y_train,X_test, y_test,reglinearie) except: pass if 'linear_model_Ridge' in model_name: #for alph in [0.1,0.4,0.5,0.6,0.7,0.8]: try: "Ordinary Least Rigde" linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge1) except: pass try: "Ordinary Least Rigde" linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge2) except: pass try: "Ordinary Least Rigde" linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge3) except: pass try: "Ordinary Least Rigde" linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge4) except: pass try: "Ordinary Least Rigde" linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge5) except: pass try: "Ordinary Least Rigde" linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge6) except: pass try: "Ordinary Least Rigde" linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge7) except: pass try: "Ordinary Least Rigde" linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge8) except: pass try: "Ordinary Least Rigde" linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge9) except: pass if 'BayesianRidge_m' in model_name: try: BayesianRidge_m(X_train, y_train,X_test, y_test,regBayesianRidge_m_T) except: pass try: BayesianRidge_m(X_train, y_train,X_test, y_test,regBayesianRidge_m_F) except: pass if 'SGDRegressor_m' in model_name: #for statut in ["squared_loss","huber","epsilon_insensitive","squared_epsilon_insensitive"]: try: SGDRegressor_m(X_train, y_train,X_test, y_test,regSGDRegressor_m1) except: pass try: SGDRegressor_m(X_train, y_train,X_test, y_test,regSGDRegressor_m2) except: pass try: SGDRegressor_m(X_train, y_train,X_test, y_test,regSGDRegressor_m3) except: pass try: SGDRegressor_m(X_train, y_train,X_test, y_test,regSGDRegressor_m4) except: pass print("tot") if 'svm_m' in model_name: #"discrimation" try: svm_m(X_train, y_train,X_test, y_test,regvm_m1) except: pass try: svm_m(X_train, y_train,X_test, y_test,regvm_m2) except: pass if 'gaussian_process_m' in model_name: #gaussian_process_m try: gaussian_process_m(X_train, y_train,X_test, y_test,gpr1) except: pass try: gaussian_process_m(X_train, y_train,X_test, y_test,gpr2) except: pass try: gaussian_process_m(X_train, y_train,X_test, y_test,gpr3) except: pass try: gaussian_process_m(X_train, y_train,X_test, y_test,gpr4) except: pass try: gaussian_process_m(X_train, y_train,X_test, y_test,gpr5) except: pass try: gaussian_process_m(X_train, y_train,X_test, y_test,gpr6) except: pass # try: # exec("RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m"+str(maxdep)+")") # except: # pass # #print(regRandomForestRegressor_m9) # print("toto") # # clf_SVR = svm.SVR() # clf_SVR.fit(X_train, y_train) # score_SVR = clf_SVR.score(X_test, y_test) # print(" SVM") # print(score_SVR) # y_pred=clf_SVR.predict(X_test) # print("gini:"+str(calcul_gini(y_test,y_pred))) # print("RMSE:"+str(mean_squared_error(y_test, y_pred))) # except: # pass # try: # clf_tree = tree.DecisionTreeRegressor() # clf_tree.fit(X_train, y_train) # score_tree = clf_tree.score(X_test, y_test) # print(" arbre de decision") # print(score_tree) # y_pred=clf_tree.predict(X_test) # print("gini:"+str(calcul_gini(y_test,y_pred))) # print("RMSE:"+str(mean_squared_error(y_test, y_pred))) # except: # pass # # "Neural network models (supervised)" # # # try: # # version classification # modelMLPC = MLPClassifier(solver='lbfgs', alpha=1e-7,hidden_layer_sizes=(10, 5)) # #to learn # modelMLPC.fit(X_train, y_train) # #to predict # print("Neural network ") # yTest_predicted =modelMLPC.predict(X_test) # print(accuracy_score(y_test,yTest_predicted)) # except: # pass #modelMLPC = MLPClassifier(solver='lbfgs', alpha=1e-7,hidden_layer_sizes=(10, 5)) #to learn #modelMLPC.fit(X_train, y_train) #to predict #yTest_predicted =modelMLPC.predict(X_test) #print(accuracy_score(y_test,yTest_predicted)) #.MLPRegressor(hidden_layer_sizes=(100, ), activation='relu', solver='adam', alpha=0.0001, batch_size='auto', learning_rate='constant', learning_rate_init=0.001, power_t=0.5, max_iter=200, shuffle=True, random_state=None, tol=0.0001, verbose=False, warm_start=False, momentum=0.9, nesterovs_momentum=True, early_stopping=False, validation_fraction=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-08, n_iter_no_change=10, max_fun=15000) #clf_SGDCl=SGDClassifier(loss="hinge", penalty="l2", max_iter=5) #clf_SGDCl.fit(X_train, y_train) #score_SGDCl = clf_SGDCl.score(X_test, y_test) #print(" arbre de decision") #print(score_SGDCl) #y_pred=clf_SGDCl.predict(X_test) #print("gini:"+str(calcul_gini(y_test,y_pred))) #print("RMSE:"+str(mean_squared_error(y_test, y_pred))) #def neural_network(XTrain, yTrain,XTest,yTest): # model = MLPClassifier(solver='lbfgs', alpha=1e-7,hidden_layer_sizes=(10, 5)) # #to learn # model.fit(XTrain, yTrain) # #to predict # yTest_predicted =model.predict(XTest) # accura=accuracy_score(yTest,yTest_predicted) # print("accuracy :"+str(accura)) # # return accura,model # print("accuracy :"+str(accura))
from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import RBF, ConstantKernel import numpy.random as rd rd.seed(42) kernel = ConstantKernel(1.0, (1e-3, 1e3)) * RBF(1, (1e-2, 1e2)) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) def f(x): return x[0] * x[1] n = 20 X = [[rd.random(), rd.random()] for _ in range(n)] Y = [f(x) for x in X] gp.fit(X, Y) x = [[0.2, 0.2]] print(gp.predict(x, return_std=True)) print(gp.predict(x, return_cov=True))
def log_marginal_likelihood(self, theta=None, eval_gradient=False): """Returns log-marginal likelihood of theta for training data. Parameters ---------- theta : array-like, shape = (n_kernel_params,) or None Kernel hyperparameters for which the log-marginal likelihood is evaluated. If None, the precomputed log_marginal_likelihood of ``self.kernel_.theta`` is returned. eval_gradient : bool, default: False If True, the gradient of the log-marginal likelihood with respect to the kernel hyperparameters at position theta is returned additionally. If True, theta must not be None. Returns ------- log_likelihood : float Log-marginal likelihood of theta for training data. log_likelihood_gradient : array, shape = (n_kernel_params,), optional Gradient of the log-marginal likelihood with respect to the kernel hyperparameters at position theta. Only returned when eval_gradient is True. """ if theta is None: if eval_gradient: raise ValueError( "Gradient can only be evaluated for theta!=None") return self.log_marginal_likelihood_value_ kernel = self.kernel_.clone_with_theta(theta) if eval_gradient: K, K_gradient = kernel(self.X_train_, eval_gradient=True) else: K = kernel(self.X_train_) # Compute log-marginal-likelihood Z and also store some temporaries # which can be reused for computing Z's gradient Z, (pi, W_sr, L, b, a) = \ self._posterior_mode(K, return_temporaries=True) if not eval_gradient: return Z # Compute gradient based on Algorithm 5.1 of GPML d_Z = np.empty(theta.shape[0]) # XXX: Get rid of the np.diag() in the next line R = W_sr[:, np.newaxis] * cho_solve((L, True), np.diag(W_sr)) # Line 7 C = solve(L, W_sr[:, np.newaxis] * K) # Line 8 # Line 9: (use einsum to compute np.diag(C.T.dot(C)))) s_2 = -0.5 * (np.diag(K) - np.einsum('ij, ij -> j', C, C)) \ * (pi * (1 - pi) * (1 - 2 * pi)) # third derivative for j in range(d_Z.shape[0]): C = K_gradient[:, :, j] # Line 11 # Line 12: (R.T.ravel().dot(C.ravel()) = np.trace(R.dot(C))) s_1 = .5 * a.T.dot(C).dot(a) - .5 * R.T.ravel().dot(C.ravel()) b = C.dot(self.y_train_ - pi) # Line 13 s_3 = b - K.dot(R.dot(b)) # Line 14 d_Z[j] = s_1 + s_2.T.dot(s_3) # Line 15 return Z, d_Z
def predict_iterative(pickle_prefix, fout): kernel = ConstantKernel() + Matern(length_scale=2, nu=3 / 2) + WhiteKernel(noise_level=1) gp = gaussian_process.GaussianProcessRegressor(kernel=kernel) probabilities_pickle_file = pickle_prefix + "probabilities.pickle" selects_pickle_file = pickle_prefix + "selects.pickle" ys_by_image_id = load_data(probabilities_pickle_file) selects_by_image_id = load_data(selects_pickle_file) max_ids = 5 num_initial_training = 200 mses = [] num_predictions = 0. num_correct = 0. num_positives = 0. num_negatives = 0. false_positives = 0. false_negatives = 0. for i, (image_id, ys) in enumerate(ys_by_image_id.iteritems()): if i > max_ids: break ys_train = ys[:num_initial_training] X = np.array(range(len(ys_train))).reshape(-1, 1) gp.fit(X, ys_train) selects = selects_by_image_id[image_id] print("-------------image-{}------------------".format(image_id)) print( "Fit image-{} with {} samples with {} average probability".format( image_id, len(ys_train), round(np.mean(ys_train), 4))) for i in range(num_initial_training + 1, len(ys)): is_selected_actual = 1 - selects[i] y_actual = ys[i] xs_test = np.array([i]).reshape(-1, 1) coin_flip = np.random.uniform(0, 1) if coin_flip < y_actual: is_selected_hypothetical = 1 else: is_selected_hypothetical = 0 y_pred, std = gp.predict(xs_test, return_std=True) is_selected_pred = predict_selection(coin_flip, y_pred, std) fout.write("prediction,{},{},{},{},{},{},{}\n".format( image_id, (xs_test[0]).tolist()[0], y_pred[0], y_actual, is_selected_actual, is_selected_hypothetical, is_selected_pred)) fout.flush() # Calculate accuracy of regression mse = get_mse(y_pred, [ys[i]]) mses.append(mse) #print("MSE: {0:5f}".format(mse)) # Calculate accuracy of selection num_predictions += 1 if is_selected_pred == is_selected_actual: num_correct += 0 if is_selected_actual: num_positives += 1 if not is_selected_pred: false_negatives += 1 else: num_negatives += 1 if not is_selected_pred: false_positives += 1 ys_train = ys[:i] X = np.array(range(len(ys_train))).reshape(-1, 1) gp.fit(X, ys_train) accuracy = num_correct / num_predictions false_positive_rate = false_positives / num_negatives false_negative_rate = false_negatives / num_positives print("Intermediate Acc:{}, FPR:{}, FNR:{}".format( accuracy, false_positive_rate, false_negative_rate)) accuracy = num_correct / num_predictions false_positive_rate = false_positives / num_negatives false_negative_rate = false_negatives / num_positives return accuracy, false_positive_rate, false_negative_rate
def plot(): i=-1#for i in range(2, 9): Xgps = Data.Xgps Ygps = Data.Ygps """plot Google Maps with GPS""" gmap = gmplot.GoogleMapPlotter( sum(Ygps[:,0])/float(len(Ygps[:,0])), sum(Ygps[:,1])/float(len(Ygps[:,1])), 18, 'AIzaSyC2I6z5RX44ZDn5z1-PiVFoEIIEVp5scKI') """ plot GPS-GPR-Prediction""" #add smoothed gps data x = np.atleast_2d(np.linspace(min(Xgps), max(Xgps), 1000)).T #RBF is a radial kernel with a length scale parameter controlling the distance # of two points influencing each other (correlation). #ConstantKernel defines a constant. #DotProduct is a linear kernel with a noise level parameter. kernel = RBF(0.01, (1e-3, 3)) *\ ConstantKernel(0.001, (1e-3, 10)) +\ WhiteKernel(1e-7, (1e-13, 1e-9))# +\ #ConstantKernel(1, (1, 10000)) *\ #DotProduct(0, (0, 180)) #noise = 0.0001 gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=3, normalize_y=True) gp.fit(Xgps, Ygps) y_pred, sigma = gp.predict(x, return_std=True) #plot some samples from posterior sample = gp.sample_y(x, n_samples=10) for s in range(10): gmap.plot(sample[:,0,s], sample[:,1,s], 'orange', edge_width=1) #plot gps data gmap.plot(Ygps[:,0], Ygps[:,1], 'cornflowerblue', edge_width=3) gmap.scatter(Ygps[:,0], Ygps[:,1], 'blue', marker=False, size=1) #plot mean posterior gmap.plot(y_pred[:,0], y_pred[:,1], 'red', edge_width=3)#, label=u'Prediction') fig, ax = plt.subplots() ax.plot(x, y_pred[:,0], 'r-', label='mean posterior') ax.plot(Xgps, Ygps[:,0], 'bo', label='original data') ax.fill(np.concatenate([x, x[::-1]]), np.concatenate([y_pred[:,0] - 1.9600 * sigma, (y_pred[:,0] + 1.9600 * sigma)[::-1]]), alpha=.3, fc='b', ec='None', label='95% confidence interval') fig.show() #============================================================================== # gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=3, normalize_y=False) # mean = Ygps.mean(0) # std = (Ygps-mean).std(0) # gp.fit(Xgps, (Ygps-mean)/std) # y_pred, sigma = gp.predict(x, return_std=True) # #plot mean posterior # gmap.plot((y_pred[:,0]*std[0]+mean[0]), (y_pred[:,1]*std[1]+mean[1]), 'red', edge_width=1)#, label=u'Prediction') # #plot some samples from posterior # sample = gp.sample_y(x, n_samples=10) # for s in range(10): # gmap.plot((sample[:,0,s]*std[0]+mean[0]), (sample[:,1,s]*std[1]+mean[1]), 'orange', edge_width=1) # #============================================================================== #gmap.polygon(np.concatenate((Ygps[:,0]-0.001, Ygps[:,0]+0.001)), np.concatenate((Ygps[:,1]+0.001,Ygps[:,1]-0.001))) """draw it and change to sattelite view""" fileName = "googlemapsplot"+str(i)+".html" print("writing result to "+fileName) gmap.draw(fileName) with open(fileName, 'r') as f: s = f.read() with open(fileName, 'w') as f: s = s.replace("MapTypeId.ROADMAP", "MapTypeId.SATELLITE") f.write(s)
def __init__(self, config, f, save=None, status=None): self.check_config(config) self.config = config self.save = save local_config = {} # self.n_jobs = self.config.n_jobs if self.config.old_type: self.pnames = self.config.pnames self.msteps = self.config.msteps self.base = self.config.pinits self.in_real = self.config.in_real # All dimensions are integers in our case, but we can optimize in real (more noise) # Here we will consider the scale as the +/- range from the initial value dimensions = [] x0 = [] # Dimensions will be automatically normalized by skopt if type(self.config.pscale) == list: for pi, si in zip(self.config.pinits, self.config.pscale): x0.append(pi) start = pi - si end = pi + si if self.in_real: dimensions.append(Real(start, end)) else: dimensions.append(Integer(start, end)) else: for pi, si, ei in zip(self.config.pinits, self.config.pmin, self.config.pmax): x0.append(pi) if self.in_real: dimensions.append(Real(si, ei)) else: dimensions.append(Integer(si, ei)) regressor = self.config.regressor normalize_y = self.config.normalize fix_noise = self.config.fix_noise games = self.config.games elo = self.config.elo isotropic = self.config.isotropic nu = self.config.nu ropoints = self.config.ropoints acq_func = self.config.acq_func simul = self.config.simul else: self.pnames = list( map(lambda p: p.name, self.config.optimization.params)) self.msteps = self.config.optimization.msteps self.base = list( map(lambda p: p.ini, self.config.optimization.params)) self.in_real = self.config.optimization.in_real # All dimensions are integers in our case, but we can optimize in real (more noise) # Here we will consider the scale as the +/- range from the initial value dimensions = [] x0 = [] # Dimensions will be automatically normalized by skopt for param in self.config.optimization.params: x0.append(param.ini) if self.in_real: dimensions.append(Real(param.min, param.max)) else: dimensions.append(Integer(param.min, param.max)) regressor = self.config.method.params.regressor normalize_y = self.config.method.params.normalize fix_noise = self.config.method.params.fix_noise games = self.config.eval.params.games elo = self.config.eval.params.elo isotropic = self.config.method.params.isotropic nu = self.config.method.params.nu ropoints = self.config.method.params.ropoints isteps = self.config.method.params.isteps acq_func = self.config.method.params.acq_func simul = self.config.eval.type == 'simul' self.is_gp = False if regressor == 'GP': self.is_gp = True # Y normalization: GP assumes mean 0, or otherwise normalize (which seems not to work well, # as the sample mean should be taken only for random points - we could calculate the mean # ourselves from the initial random points - not done for now) # Yet is normalization the way to go... # The noise level is fix in our case, as we play a fixed number of games per step # (exception: initial / reference point, with noise = 0 - we ignore this) # It depends of number of games per step and the loss function (elo/elowish) # The formula below is an ELO error approximation for the confidence interval of 95%, # which lies by about 2 sigma - we can compute sigma of the error if fix_noise: noise_level_bounds = 'fixed' sigma = 250. / math.sqrt(games) if not elo: sigma = sigma * math.log(10) / 400. noise_level = sigma * sigma else: if elo and not normalize_y: noise_level_bounds = (1e-2, 1e4) noise_level = 10 else: noise_level_bounds = (1e-6, 1e0) noise_level = 0.01 # Length scales: because skopt normalizes the dimensions automatically, it is unclear # how to proceed here, as the kernel does not know about that normalization... length_scale_bounds = (1e-3, 1e4) length_scale = 1 # Isotropic or anisotropic kernel if not isotropic: length_scale = length_scale * np.ones(len(dimensions)) # Matern kernel with configurable parameter nu (1.5 = once differentiable functions), # white noise kernel and a constant kernel for mean estimatation kernel = \ ConstantKernel() \ + WhiteKernel(noise_level=noise_level, noise_level_bounds=noise_level_bounds) \ + 1.0 * Matern(nu=nu, length_scale=length_scale, \ length_scale_bounds=length_scale_bounds) # We put alpha=0 because we count in the kernel for the noise # n_restarts_optimizer is important to find a good fit! (but it costs time) rgr = GaussianProcessRegressor(kernel=kernel, alpha=0.0, normalize_y=normalize_y, n_restarts_optimizer=ropoints) else: rgr = regressor # When we start to use the regressor, we should have enough random points # for a good space exploration if isteps == 0: n_initial_points = max(self.msteps // 10, len(dimensions) + 1) else: n_initial_points = isteps self.optimizer = Optimizer( dimensions=dimensions, base_estimator=rgr, acq_func=acq_func, acq_optimizer='sampling', # without this I got this error: # grad = self.kernel_.gradient_x(X[0], self.X_train_) # AttributeError: 'Sum' object has no attribute 'gradient_x' n_initial_points=n_initial_points, initial_point_generator='lhs', # latin hypercube, just a try n_jobs=-1, # use all cores in the estimator model_queue_size=2) self.done = False if status is None: # When we start, we know that the initial point is the reference, mark it with 0 # Unless we simulate! if simul: self.theta = None self.best = None self.xi = [] self.yi = [] else: self.theta = x0 self.best = 0 self.xi = [x0] self.yi = [0] self.optimizer.tell(x0, 0) else: self.theta = status['theta'] self.best = status['best'] self.xi = status['xi'] self.yi = status['yi'] # Here: because we maximize (while the bayes optimizer minimizes), negate! self.optimizer.tell(self.xi, list(map(lambda y: -y, self.yi))) # Because we added the reference result, we have now one measurement more than steps self.step = len(self.xi) self.func = f
y = pd.read_csv('weeks.csv', header=0) df = df_1.transpose() df = df[1:] X = df median_pred = pd.DataFrame({"patient_id": X.index}) y = pd.concat([median_pred, y], axis=1) y['patient_id'] = y['patient_id'].astype(str) Y = y.set_index('patient_id') Y = Y["weeks"] Y X["p"] = [x.split("_")[0] for x in X.index] y["p"] = [x.split("_")[0] for x in Y.index] ker_rbf = ConstantKernel(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)) ker_rq = ConstantKernel(1.0, (1e-3, 1e3)) * RationalQuadratic(alpha=0.1, length_scale=1) kernel_list = [ker_rbf, ker_rq] param_grid = { "kernel": kernel_list, "alpha": [1e1], "optimizer": ["fmin_l_bfgs_b"], "n_restarts_optimizer": [10, 15, 20, 25] } prediction = [] i = 0 scaler = MinMaxScaler(feature_range=(0, 1)) sc = StandardScaler()
def __init__(self, sample, data, kernel=None, noise=False, global_optimizer=True): r"""Create the predictor. Uses sample and data to construct a predictor using Gaussian Process. Input is to be normalized before and depending on the number of parameters, the kernel is adapted to be anisotropic. :attr:`self.data` contains the predictors as a list(array) of the size of the `ouput`. A predictor per line of `data` is created. This leads to a line of predictors that predicts a new column of `data`. If :attr:`noise` is a float, it will be used as :attr:`noise_level` by :class:`sklearn.gaussian_process.kernels.WhiteKernel`. Otherwise, if :attr:`noise` is ``True``, default values are use for the WhiteKernel. If :attr:`noise` is ``False``, no noise is added. A multiprocessing strategy is used: 1. Create a process per mode, do not create if only one, 2. Create `n_restart` (3 by default) processes by process. In the end, there is :math:`N=n_{restart} \times n_{modes})` processes. If there is not enought CPU, :math:`N=\frac{n_{cpu}}{n_{restart}}`. :param array_like sample: Sample used to generate the data (n_samples, n_features). :param array_like data: Observed data (n_samples, n_features). :param kernel: Kernel from scikit-learn. :type kernel: :class:`sklearn.gaussian_process.kernels`.*. :param float/bool noise: Noise used into kriging. :param bool global_optimizer: Whether to do global optimization or gradient based optimization to estimate hyperparameters. """ sample = np.atleast_2d(sample) dim = sample.shape[1] self.model_len = data.shape[1] if self.model_len == 1: data = data.ravel() if kernel is not None: self.kernel = kernel else: # Define the model settings l_scale = (1.0, ) * dim self.scale_bounds = [(0.01, 100)] * dim self.kernel = ConstantKernel() * Matern( length_scale=l_scale, length_scale_bounds=self.scale_bounds) # Add a noise on the kernel using WhiteKernel if noise: if isinstance(noise, bool): noise = WhiteKernel() else: noise = WhiteKernel(noise_level=noise) self.kernel += noise # Global optimization args_optim = {'kernel': self.kernel, 'normalize_y': True} if global_optimizer: args_optim.update({ 'optimizer': self._optim_evolution, 'n_restarts_optimizer': 0 }) self.n_restart = 3 else: args_optim.update({'n_restarts_optimizer': 10 * dim}) self.n_restart = 1 # Define the CPU multi-threading/processing strategy n_cpu_system = cpu_system() self.n_cpu = self.n_restart * self.model_len if (n_cpu_system // (self.n_restart * self.model_len) < 1)\ or (self.n_cpu > n_cpu_system // self.n_restart): self.n_cpu = n_cpu_system // self.n_restart self.n_cpu = 1 if self.n_cpu == 0 else self.n_cpu def model_fitting(column): """Fit an instance of :class:`sklearn.GaussianProcessRegressor`.""" gp = GaussianProcessRegressor(**args_optim) with warnings.catch_warnings(): warnings.simplefilter("ignore") data = gp.fit(sample, column) hyperparameter = np.exp(gp.kernel_.theta) # Convergence check with bounds only when kernel not user defined if kernel is None: hyper_bounds = all([ i[0] < j < i[1] for i, j in zip(self.scale_bounds, hyperparameter[1:dim + 1]) ]) if not hyper_bounds: self.logger.warning("Hyperparameters optimization not " "converged: {}".format(gp.kernel_)) return data, hyperparameter # Create a predictor per data, parallelize if several data if self.model_len > 1: pool = NestedPool(self.n_cpu) results = pool.imap(model_fitting, data.T) results = list(results) pool.terminate() else: results = [model_fitting(data)] # Gather results self.gp_models, self.hyperparameter = zip(*results) self.logger.debug("Kernels:\n{}".format( [gp.kernel_ for gp in self.gp_models]))
def main(_): num_steps_autoencoder = 0 if FLAGS.uniform_weights else TRAINING_STEPS training_df = pd.read_csv(FLAGS.training_data_path, header=0, sep=',') testing_df = pd.read_csv(FLAGS.testing_data_path, header=0, sep=',') validation_df = pd.read_csv(FLAGS.validation_data_path, header=0, sep=',') train_labels = training_df['label'] validation_labels = validation_df['label'] test_labels = testing_df['label'] train_population = training_df['population'] train_features = training_df[FEATURES] validation_features = validation_df[FEATURES] test_features = testing_df[FEATURES] train_wqs = training_df['racePctWhite_quantile'] validation_wqs = validation_df['racePctWhite_quantile'] test_wqs = testing_df['racePctWhite_quantile'] tf.reset_default_graph() x = tf.placeholder(tf.float32, shape=(None, len(FEATURES)), name='x') y = tf.placeholder(tf.float32, shape=(None, OUTPUT_DIM), name='y') population = tf.placeholder(tf.float32, shape=(None, OUTPUT_DIM), name='population') xy = tf.concat([x, y], axis=1) autoencoder_layer1 = tf.layers.dense(inputs=xy, units=10, activation=tf.sigmoid) autoencoder_embedding_layer = tf.layers.dense(inputs=autoencoder_layer1, units=EMBEDDING_DIM, activation=tf.sigmoid) autoencoder_layer3 = tf.layers.dense(inputs=autoencoder_embedding_layer, units=10, activation=tf.sigmoid) autoencoder_out_x = tf.layers.dense(inputs=autoencoder_layer3, units=len(FEATURES)) autoencoder_out_y_logits = tf.layers.dense(inputs=autoencoder_layer3, units=OUTPUT_DIM) autoencoder_y_loss = tf.losses.hinge_loss(labels=y, logits=autoencoder_out_y_logits) autoencoder_x_loss = tf.losses.mean_squared_error( labels=x, predictions=autoencoder_out_x) autoencoder_loss = autoencoder_x_loss + autoencoder_y_loss autoencoder_optimizer = tf.train.AdamOptimizer(LEARNING_RATE).minimize( autoencoder_loss) parallel_logits = [] parallel_losses = [] parallel_optimizers = [] parallel_alphas = tf.placeholder(tf.float32, shape=(NUM_PARALLEL_ALPHAS, EMBEDDING_DIM), name='parallel_alphas') unstack_parallel_alphas = tf.unstack(parallel_alphas, axis=0) embedding = tf.placeholder(tf.float32, shape=(None, EMBEDDING_DIM), name='embedding') with tf.variable_scope('classifiers'): for alpha_index in range(NUM_PARALLEL_ALPHAS): logits = classifier(x) alpha = tf.reshape(unstack_parallel_alphas[alpha_index], shape=[EMBEDDING_DIM, 1]) optimizer, loss = optimization(logits, y, population, embedding, alpha) parallel_logits.append(logits) parallel_losses.append(loss) parallel_optimizers.append(optimizer) init = tf.global_variables_initializer() classifiers_init = tf.variables_initializer( tf.global_variables(scope='classifiers')) kernel = RBF(length_scale=FLAGS.sampling_radius, length_scale_bounds=(FLAGS.sampling_radius * 1e-3, FLAGS.sampling_radius * 1e3)) * ConstantKernel(1.0, (1e-3, 1e3)) alphas = np.zeros(shape=(0, EMBEDDING_DIM)) validation_metrics = [] test_metrics = [] with tf.Session() as sess: sess.run(init) # Training autoencoder for _ in range(num_steps_autoencoder): batch_index = random.sample(range(len(train_labels)), BATCH_SIZE) batch_x = train_features.iloc[batch_index, :].values batch_y = train_labels.iloc[batch_index].values.reshape( BATCH_SIZE, 1) _, _ = sess.run([autoencoder_optimizer, autoencoder_loss], feed_dict={ x: batch_x, y: batch_y, }) # GetCandidatesAlpha (Algorithm 2 in paper) for alpha_batch_index in range(NUM_ALPHA_BATCHES): sess.run(classifiers_init) if FLAGS.uniform_weights: alpha_batch = np.zeros(shape=(NUM_PARALLEL_ALPHAS, EMBEDDING_DIM)) elif alpha_batch_index == 0: # We first start uniformly. alpha_batch = sample_from_ball( size=(NUM_PARALLEL_ALPHAS, EMBEDDING_DIM), sampling_radius=FLAGS.sampling_radius) else: # Use UCB to generate candidates. alpha_batch = np.zeros(shape=(0, EMBEDDING_DIM)) sample_alphas = np.copy(alphas) sample_validation_metrics = [m[0] for m in validation_metrics] candidates = sample_from_ball( size=(10000, EMBEDDING_DIM), sampling_radius=FLAGS.sampling_radius) for alpha_index in range(NUM_PARALLEL_ALPHAS): gp = GaussianProcessRegressor( kernel=kernel, alpha=1e-1).fit(sample_alphas, sample_validation_metrics) metric_mles, metric_stds = gp.predict(candidates, return_std=True) metric_lcbs = metric_mles - 1.0 * metric_stds best_index = np.argmin(metric_lcbs) best_alpha = [candidates[best_index]] best_alpha_metric_ucb = metric_mles[best_index] \ + 1.0 * metric_stds[best_index] alpha_batch = np.concatenate([alpha_batch, best_alpha]) # Add candidate to the GP, assuming the metric observation is the LCB. sample_alphas = np.concatenate([sample_alphas, best_alpha]) sample_validation_metrics.append(best_alpha_metric_ucb) # Training classifiers for _ in range(TRAINING_STEPS): batch_index = random.sample(range(len(train_labels)), BATCH_SIZE) batch_x = train_features.iloc[batch_index, :].values batch_y = train_labels.iloc[batch_index].values.reshape( BATCH_SIZE, 1) batch_population = train_population.iloc[ batch_index].values.reshape(BATCH_SIZE, 1) batch_embedding = sess.run(autoencoder_embedding_layer, feed_dict={ x: batch_x, y: batch_y, }) _, _ = sess.run( [parallel_optimizers, parallel_losses], feed_dict={ x: batch_x, y: batch_y, population: batch_population, embedding: batch_embedding, parallel_alphas: alpha_batch, }) parallel_train_logits = sess.run(parallel_logits, feed_dict={ x: train_features.values, y: train_labels.values.reshape( len(train_labels), 1), }) alphas = np.concatenate([alphas, alpha_batch]) parallel_validation_logits = sess.run( parallel_logits, feed_dict={ x: validation_features.values, y: validation_labels.values.reshape(len(validation_labels), 1), }) parallel_test_logits = sess.run(parallel_logits, feed_dict={ x: test_features.values, y: test_labels.values.reshape( len(test_labels), 1), }) parallel_thresholds = [ find_threshold(train_labels, train_logits, train_wqs, FLAGS.post_shift) for train_logits in parallel_train_logits ] logits_thresholds = zip(parallel_validation_logits, parallel_thresholds) parallel_validation_metrics = [ metrics(validation_labels, logits, validation_wqs, thresholds) for (logits, thresholds) in logits_thresholds ] validation_metrics.extend(parallel_validation_metrics) parallel_test_metrics = [ metrics(test_labels, test_logits, test_wqs, thresholds) for (test_logits, thresholds ) in zip(parallel_test_logits, parallel_thresholds) ] test_metrics.extend(parallel_test_metrics) best_observed_index = np.argmin([m[0] for m in validation_metrics]) print('[metric] validation_acc={}'.format( validation_metrics[best_observed_index][0])) print('[metric] validation_violation={}'.format( validation_metrics[best_observed_index][1])) print('[metric] test_acc={}'.format(test_metrics[best_observed_index][0])) print('[metric] test_violation={}'.format( test_metrics[best_observed_index][1])) return 0