ConstantKernel, WhiteKernel, PairwiseKernel, KernelOperator,
Exponentiation)
from sklearn.base import clone
from sklearn.utils.testing import (assert_equal, assert_almost_equal,
assert_not_equal, assert_array_equal,
assert_array_almost_equal)
X = np.random.RandomState(0).normal(0, 1, (5, 2))
Y = np.random.RandomState(0).normal(0, 1, (6, 2))
kernel_white = RBF(length_scale=2.0) + WhiteKernel(noise_level=3.0)
kernels = [
RBF(length_scale=2.0),
RBF(length_scale_bounds=(0.5, 2.0)),
ConstantKernel(constant_value=10.0),
2.0 * RBF(length_scale=0.33, length_scale_bounds="fixed"), 2.0 *
RBF(length_scale=0.5), kernel_white, 2.0 * RBF(length_scale=[0.5, 2.0]),
2.0 * Matern(length_scale=0.33, length_scale_bounds="fixed"),
2.0 * Matern(length_scale=0.5, nu=0.5),
2.0 * Matern(length_scale=1.5, nu=1.5),
2.0 * Matern(length_scale=2.5, nu=2.5),
2.0 * Matern(length_scale=[0.5, 2.0], nu=0.5),
3.0 * Matern(length_scale=[2.0, 0.5], nu=1.5),
4.0 * Matern(length_scale=[0.5, 0.5], nu=2.5),
RationalQuadratic(length_scale=0.5, alpha=1.5),
ExpSineSquared(length_scale=0.5, periodicity=1.5),
DotProduct(sigma_0=2.0),
DotProduct(sigma_0=2.0)**2
]
for metric in PAIRWISE_KERNEL_FUNCTIONS:
x_pred.columns = x.columns
# オートスケーリング
autoscaled_y = (y - y.mean()) / y.std()
autoscaled_x = (x - x.mean()) / x.std()
autoscaled_x_pred = (x_pred - x.mean()) / x.std()
# データ確認
pd.concat([autoscaled_y, autoscaled_x], axis=1)
autoscaled_x_pred
# 5 カーネルの設定 -----------------------------------------------------------
# カーネル 11 種類
kernels = [
ConstantKernel() * DotProduct() + WhiteKernel(),
ConstantKernel() * RBF() + WhiteKernel(),
ConstantKernel() * RBF() + WhiteKernel() + ConstantKernel() * DotProduct(),
ConstantKernel() * RBF(np.ones(x.shape[1])) + WhiteKernel(),
ConstantKernel() * RBF(np.ones(x.shape[1])) + WhiteKernel() +
ConstantKernel() * DotProduct(),
ConstantKernel() * Matern(nu=1.5) + WhiteKernel(),
ConstantKernel() * Matern(nu=1.5) + WhiteKernel() +
ConstantKernel() * DotProduct(),
ConstantKernel() * Matern(nu=0.5) + WhiteKernel(),
ConstantKernel() * Matern(nu=0.5) + WhiteKernel() +
ConstantKernel() * DotProduct(),
ConstantKernel() * Matern(nu=2.5) + WhiteKernel(),
ConstantKernel() * Matern(nu=2.5) + WhiteKernel() +
ConstantKernel() * DotProduct()
]
def main():
if len(sys.argv) < 2:
print('syntax:\t%s <csv file>' % sys.argv[0])
return
co2data = pd.read_csv(sys.argv[1])
print(co2data.head())
print(co2data.isnull().any())
co2data_full = co2data.copy(deep=True)
co2data = co2data.dropna()
print('referencing dates from 1950\ndays as % of the year')
x = np.array(co2data['Decimal Date'] - 1950)
m = 40 #np.mean(x)
y = np.array(co2data['Carbon Dioxide (ppm)'])
x_learn = featurize(x, m)
y_learn = [val for val in y]
x_train, x_test, x_validate, y_train, y_test, y_validate, x_oob, y_oob = katiesplit(
x_learn, y_learn)
quadmod = linear_model.LinearRegression().fit(x_train, y_train)
print("Accuracy (test): ", quadmod.score(x_test, y_test))
print("Accuracy (validate): ", quadmod.score(x_validate, y_validate))
np.savetxt('%s.original' % sys.argv[1],
np.column_stack((x, x_learn, y_learn)),
fmt='%.2f')
x_learn = refeaturize(x, m)
y_learn -= quadmod.predict(featurize(x, m))
x_train, x_test, x_validate, y_train, y_test, y_validate, x_oob, y_oob = katiesplit(
x_learn, y_learn)
pertmod = linear_model.LinearRegression().fit(x_train, y_train)
print("Perturbation Accuracy (test): ", pertmod.score(x_test, y_test))
print("Perturbation Accuracy (validate): ",
pertmod.score(x_validate, y_validate))
pred_arg = np.linspace(0, 80, 2561)
x_pred = refeaturize(pred_arg, m)
y_pred = pertmod.predict(x_pred) + quadmod.predict(featurize(pred_arg, m))
np.savetxt('%s.predictions' % sys.argv[1],
np.column_stack((pred_arg, x_pred, y_pred)),
fmt='%.2f')
''' =============== Gaussian process section =================== '''
x_learn = x.copy()
y_learn = [val for val in y]
x_train, x_test, x_validate, y_train, y_test, y_validate, x_oob, y_oob = katiesplit(
x_learn, y_learn)
k1 = 66.0**2 * RBF(length_scale=67.0) # long term smooth rising trend
k2 = 2.4**2 * RBF(length_scale=90.0) * ExpSineSquared(
length_scale=1.3, periodicity=1.0) # seasonal component
k3 = 0.66**2 * RationalQuadratic(length_scale=1.2,
alpha=0.78) # medium term irregularity
k4 = 0.18**2 * RBF(length_scale=0.134) + WhiteKernel(noise_level=0.19**
2) # noise terms
kernel_gpml = k1 + k2 + k3 + k4
gp = GaussianProcessRegressor(kernel=kernel_gpml,
alpha=0,
optimizer=None,
normalize_y=True)
gp.fit(
np.asarray(x_train).reshape(-1, 1),
np.asarray(y_train).reshape(-1, 1))
print("GPML kernel: %s" % gp.kernel_)
print("Log-marginal-likelihood: %.3f" %
gp.log_marginal_likelihood(gp.kernel_.theta))
print(
'GP score (test): ',
gp.score(
np.asarray(x_test).reshape(-1, 1),
np.asarray(y_test).reshape(-1, 1)))
print(
'GP score (validate): ',
gp.score(
np.asarray(x_validate).reshape(-1, 1),
np.asarray(y_validate).reshape(-1, 1)))
# Kernel with reduced parameters
#k1 = 50.0**2 * RBF(length_scale=50.0) # long term smooth rising trend
k1 = 50.0**2 * RationalQuadratic(
length_scale=50.0, alpha=1.0) # long term smooth rising trend
#k2 = 2.0**2 * RBF(length_scale=100.0) * ExpSineSquared(length_scale=1.0, periodicity=1.0, periodicity_bounds="fixed") # seasonal component
k2 = 2.0**2 * ExpSineSquared(
length_scale=1.0, periodicity=1.0,
periodicity_bounds=(.9, 1.1)) # seasonal component
#k3 = 10**2 * RationalQuadratic(length_scale=10.0, alpha=1.0) # medium term irregularities
k4 = 0.1**2 * RBF(length_scale=0.1) + WhiteKernel(
noise_level=0.1**2, noise_level_bounds=(1e-3, np.inf)) # noise terms
k5 = ConstantKernel(constant_value=10000,
constant_value_bounds="fixed") # baseline shift
kernel = k1 + k2 + k4 + k5
gp_me = GaussianProcessRegressor(kernel=kernel, alpha=0, normalize_y=True)
gp_me.fit(
np.asarray(x_train).reshape(-1, 1),
np.asarray(y_train).reshape(-1, 1))
print("\nLearned kernel: %s" % gp_me.kernel_)
print("Log-marginal-likelihood: %.3f" %
gp_me.log_marginal_likelihood(gp_me.kernel_.theta))
x_pred = pred_arg[:, np.newaxis]
y_pred, y_std = gp.predict(np.asarray(x_pred).reshape(-1, 1),
return_std=True)
y_pred_me, y_std_me = gp_me.predict(np.asarray(x_pred).reshape(-1, 1),
return_std=True)
print(
'GP score (test): ',
gp_me.score(
np.asarray(x_test).reshape(-1, 1),
np.asarray(y_test).reshape(-1, 1)))
print(
'GP score (validate): ',
gp_me.score(
np.asarray(x_validate).reshape(-1, 1),
np.asarray(y_validate).reshape(-1, 1)))
np.savetxt('%s.gp_predictions' % sys.argv[1],
np.column_stack((x_pred, y_pred, y_std, y_pred_me, y_std_me)),
fmt='%.2f')
print("Now doing compound quadratic linregression, then reduced GP\n\n")
print("... actually going to skip this for now")
return
x = np.array(co2data['Decimal Date'] - 1950)
m = 40 #np.mean(x)
y = np.array(co2data['Carbon Dioxide (ppm)'])
x_learn = x.copy()
y_learn = [val for val in y]
x_train, x_test, x_validate, y_train, y_test, y_validate, x_oob, y_oob = katiesplit(
x_learn, y_learn)
quadmod = linear_model.LinearRegression().fit(featurize(x_oob, m), y_oob)
k1 = 2.0**2 * RBF(length_scale=100.0) * ExpSineSquared(
length_scale=1.0, periodicity=1.0,
periodicity_bounds="fixed") # seasonal component
k2 = 0.5**2 * RationalQuadratic(length_scale=1.0,
alpha=1.0) # medium term irregularities
k3 = 0.1**2 * RBF(length_scale=0.1) + WhiteKernel(
noise_level=0.1**2, noise_level_bounds=(1e-3, np.inf)) # noise terms
kernel_compound = k1 + k2 + k3
gp = GaussianProcessRegressor(kernel=kernel_compound,
alpha=0,
normalize_y=True)
gp.fit(
np.asarray(x_train).reshape(-1, 1),
np.asarray(y_train -
quadmod.predict(featurize(x_train, m)).reshape(-1, 1)))
print("\nLearned kernel: %s" % gp.kernel_)
print("Log-marginal-likelihood: %.3f" %
gp.log_marginal_likelihood(gp.kernel_.theta))
y_pred = gp.predict(np.asarray(x_pred).reshape(-1, 1), return_std=False)
y_qm = quadmod.predict(featurize(x_pred, m))
y_pred = np.asarray(y_pred).reshape(-1, 1) + y_qm
#print('GP score (test): ', gp.score(np.asarray(x_test).reshape(-1,1), np.asarray(y_test).reshape(-1,1) - quadmod.predict(featurize(x_test,m)) ))
#print('GP score (validate): ', gp.score(np.asarray(x_validate).reshape(-1,1), np.asarray(y_validate - quadmod.predict(featurize(x_validate,m))).reshape(-1,1) ))
np.savetxt('%s.quad_gp_predictions' % sys.argv[1],
np.column_stack((x_pred, y_pred)),
fmt='%.2f')
return
vec = CountVectorizer()
X = vec.fit_transform(sample)
pd.DataFrame(X.toarray(), columns=vec.get_feature_names())
from sklearn.feature_extraction.text import TfidfVectorizer
vec = TfidfVectorizer()
X = vec.fit_transform(sample)
pd.DataFrame(X.toarray(), columns=vec.get_feature_names())
from sklearn.model_selection import train_test_split
Xtrain, Xtest, ytrain, ytest = train_test_split(X, alphas, random_state=1)
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import ConstantKernel, RBF
rbf = ConstantKernel(1.0) * RBF(length_scale=1.0)
gpr = GaussianProcessRegressor(kernel=rbf, alpha=1e-8)
gpr.fit(Xtrain.toarray(), ytrain)
mu_s, cov_s = gpr.predict(Xtest.toarray(), return_cov=True)
print("Tfid vectorizer results: ", np.corrcoef(ytest, mu_s))
# r = 0.683
bgrmvector = CountVectorizer(ngram_range=(2, 2),
token_pattern=r'\b\w+\b',
min_df=1)
X = bgrmvector.fit_transform(sample)
pd.DataFrame(X.toarray(), columns=bgrmvector.get_feature_names())
Xtrain, Xtest, ytrain, ytest = train_test_split(X, alphas, random_state=1)
rbf = ConstantKernel(1.0) * RBF(length_scale=1.0)
gpr = GaussianProcessRegressor(kernel=rbf, alpha=1e-8)
gpr.fit(Xtrain.toarray(), ytrain)
mu_s, cov_s = gpr.predict(Xtest.toarray(), return_cov=True)
'k',
linewidth=2)
plt.xlabel('Ratio of the number of X-variables')
plt.ylabel('RMSE of OOB')
plt.show()
optimal_random_forest_x_variables_rate = random_forest_x_variables_rates[
np.where(rmse_oob_all == np.min(rmse_oob_all))[0][0]]
regression_model = RandomForestRegressor(
n_estimators=random_forest_number_of_trees,
max_features=int(
max(
math.ceil(autoscaled_x_train.shape[1] *
optimal_random_forest_x_variables_rate), 1)),
oob_score=True)
elif method == 'gp': # Gaussian process
regression_model = GaussianProcessRegressor(ConstantKernel() * RBF() +
WhiteKernel(),
alpha=0)
elif method == 'lgb': # LightGBM
import lightgbm as lgb
regression_model = lgb.LGBMRegressor()
elif method == 'xgb': # XGBoost
import xgboost as xgb
regression_model = xgb.XGBRegressor()
elif method == 'gbdt': # scikit-learn
from sklearn.ensemble import GradientBoostingRegressor
regression_model = GradientBoostingRegressor()
regression_model.fit(autoscaled_x_train, autoscaled_y_train)
[1e-3, 1e-2, 0.1, 0.5, 0.9, 2]
}),
(RadiusNeighborsClassifier, {
'radius': neighbor_radius,
'algo': neighbor_algo,
'leaf_size': neighbor_leaf_size,
'metric': neighbor_metric,
'weights': ['uniform', 'distance'],
'p': [1, 2],
'outlier_label': [-1]
})]
gaussianprocess_models_n_params = [(GaussianProcessClassifier, {
'warm_start':
warm_start,
'kernel': [RBF(), ConstantKernel(),
DotProduct(),
WhiteKernel()],
'max_iter_predict': [500],
'n_restarts_optimizer': [3],
})]
bayes_models_n_params = [(GaussianNB, {})]
nn_models_n_params = [(MLPClassifier, {
'hidden_layer_sizes': [(16, ), (64, ), (100, ), (32, 32)],
'activation': ['identity', 'logistic', 'tanh', 'relu'],
'alpha':
alpha,
'learning_rate':
learning_rate,
axs[1].set_title("Samples from posterior distribution")
fig.suptitle("Periodic kernel", fontsize=18)
plt.tight_layout()
# %%
print(f"Kernel parameters before fit:\n{kernel})")
print(f"Kernel parameters after fit: \n{gpr.kernel_} \n"
f"Log-likelihood: {gpr.log_marginal_likelihood(gpr.kernel_.theta):.3f}")
# %%
# Dot product kernel
# ..................
from sklearn.gaussian_process.kernels import ConstantKernel, DotProduct
kernel = ConstantKernel(0.1, (0.01, 10.0)) * (DotProduct(
sigma_0=1.0, sigma_0_bounds=(0.1, 10.0))**2)
gpr = GaussianProcessRegressor(kernel=kernel, random_state=0)
fig, axs = plt.subplots(nrows=2, sharex=True, sharey=True, figsize=(10, 8))
# plot prior
plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[0])
axs[0].set_title("Samples from prior distribution")
# plot posterior
gpr.fit(X_train, y_train)
plot_gpr_samples(gpr, n_samples=n_samples, ax=axs[1])
axs[1].scatter(X_train[:, 0],
y_train,
color="red",
from matplotlib import pyplot as plt
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import (RBF, Matern, RationalQuadratic,
ExpSineSquared, DotProduct,
ConstantKernel)
kernels = [
1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)),
1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1),
1.0 * ExpSineSquared(length_scale=1.0,
periodicity=3.0,
length_scale_bounds=(0.1, 10.0),
periodicity_bounds=(1.0, 10.0)),
ConstantKernel(0.1, (0.01, 10.0)) *
(DotProduct(sigma_0=1.0, sigma_0_bounds=(0.1, 10.0))**2),
1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), nu=1.5)
]
for fig_index, kernel in enumerate(kernels):
# Specify Gaussian Process
gp = GaussianProcessRegressor(kernel=kernel)
# Plot prior
plt.figure(fig_index, figsize=(8, 8))
plt.subplot(2, 1, 1)
X_ = np.linspace(0, 5, 100)
y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True)
plt.plot(X_, y_mean, 'k', lw=3, zorder=9)
plt.fill_between(X_, y_mean - y_std, y_mean + y_std, alpha=0.2, color='k')
y_train = sc_y.transform(y_train)
# transform test dataset
y_test = sc_y.transform(y_test)
print('Training Features Shape:', x_train.shape)
print('Training Labels Shape:', y_train.shape)
print('Testing Features Shape:', x_test.shape)
print('Testing Labels Shape:', y_test.shape)
hyper_params = [{
'alpha': (1e-10, ),
#'alpha': (1e-2, 1e-1, 1e0, 1e1, 1e2,),
#'n_restarts_optimizer': (0,1,10,100,),
'n_restarts_optimizer': (0, ),
#'kernel': (DotProduct() + WhiteKernel(),),
'kernel': (ConstantKernel(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)), ),
}]
# 'kernel': ([1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)),
# 1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1),
# 1.0 * ExpSineSquared(length_scale=1.0, periodicity=3.0,
# length_scale_bounds=(0.1, 10.0),
# periodicity_bounds=(1.0, 10.0)),
# ConstantKernel(0.1, (0.01, 10.0))
# * (DotProduct(sigma_0=1.0, sigma_0_bounds=(0.1, 10.0)) ** 2),
# 1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), nu=1.5)],), }]
# "kernel": ([ExpSineSquared(l, p)
# for l in np.logspace(-2, 2, 10)
# for p in np.logspace(0, 2, 10)],
# [DotProduct() + WhiteKernel()],
def add_parameter_ui(classifier_name):
param = dict()
if classifier_name == "KNN":
k = st.sidebar.slider("K", 1, 15)
param["K"] = k
elif classifier_name == "SVM":
C = st.sidebar.slider("C", 0.01, 10.0)
param["C"] = C
elif classifier_name == "Random Forest":
max_depth = st.sidebar.slider("Max depth", 2, 16)
estimator = st.sidebar.slider("estimator", 1, 100)
param["max_depth"] = max_depth
param["estimator"] = estimator
elif classifier_name == "BaggingClassifier":
param["base_estimator"] = SVC()
n_estimator = st.sidebar.slider("n_estimator", 1, 20)
max_sample = st.sidebar.slider("Max Sample", 0.1, 1.0)
param["n_estimator"] = n_estimator
param["max_sample"] = max_sample
elif classifier_name == "DecisionTreeClassifier":
max_depth = st.sidebar.slider("Max depth", 2, 16)
leaf = st.sidebar.slider("Leaf Nodes", 1, 100)
param["max_depth"] = max_depth
param["Leaf"] = leaf
elif classifier_name == "ExtraTreeClassifier":
max_depth = st.sidebar.slider("Max depth", 2, 16)
leaf = st.sidebar.slider("Leaf Nodes", 1, 100)
min_sample_split = st.sidebar.slider("min_sample_split", 0.1, 5.0)
param["max_depth"] = max_depth
param["Leaf"] = leaf
if min_sample_split <= 1:
param["min_sample_split"] = float(min_sample_split)
elif min_sample_split > 1 and min_sample_split < 2:
param["min_sample_split"] = int(min_sample_split) + 1
else:
param["min_sample_split"] = int(min_sample_split)
elif classifier_name == "GaussianProcessClassifier":
n_restarts_optimizer = st.sidebar.slider("n_restarts_optimizer", 0, 10)
max_iter_predict = st.sidebar.slider("max_iter_predict", 50, 100)
l = [
1 * RBF(), 1 * DotProduct(), 1 * ConstantKernel(),
1 * RationalQuadratic()
]
kernal = st.sidebar.selectbox("Kernal", l)
param["optimizer"] = n_restarts_optimizer
param["predict"] = max_iter_predict
param["kernal"] = kernal
elif classifier_name == "LinearSVC":
loss = st.sidebar.selectbox("loss", ('hinge', 'squared_hinge'))
C = st.sidebar.slider("C", 0.1, 5.0)
param["loss"] = loss
param["c"] = C
elif classifier_name == "NuSVC":
kernel = st.sidebar.selectbox(
"kernel", ('linear', 'poly', 'rbf', 'sigmoid', 'precomputed'))
degree = st.sidebar.slider("degree", 0, 5)
gamma = st.sidebar.selectbox("Gamma", ('scale', 'auto'))
param["kernel"] = kernel
param["degree"] = degree
param["gamma"] = gamma
elif classifier_name == "AdaBoostClassifier":
n_estimator = st.sidebar.slider("n_estimators", 10, 100)
learning_rate = st.sidebar.slider("Learning Rate", 0.01, 1.00)
algorithm = st.sidebar.selectbox("Algorithm", ('SAMME', 'SAMME.R'))
param["estimators"] = n_estimator
param["learning"] = learning_rate
param["algo"] = algorithm
return param
# strategy
strategy = SRBFStrategy(lb, ub)
# uncertainty information
mu = 0.0
sigma = np.sqrt(0.01)
p = lambda num_pts: np.random.normal(mu, sigma, (num_pts, dim))
# Monte Carlo parameters
num_points_MC = 5000
# Mean Variance weighting
eta = 0.5
# surrogate
kernel = ConstantKernel(1, (1e-3, 1e3)) * RBF(1, (0.1, 100)) + \
WhiteKernel(1e-3, (1e-6, 1e-2))
surrogate = MeanVariance(kernel, p, eta, num_points_MC)
# initialize the problem
problem = BayesianOptimization(f, dim, max_evals, exp_design, strategy,
surrogate, lb, ub)
# solve it
xopt, fopt = problem.minimize()
#=============================================================
# Plot
#=============================================================
# test points
Ntest = 300
def special_conversions(self, classifier_params):
"""
TODO: Make this logic into subclasses
ORRRR, should make each enumerator handle it in a static function
something like:
@staticmethod
def ParamsTransformation(params_dict):
# does classifier-specific changes
return params_dict
"""
# do special converstions
### RF ###
if "n_jobs" in classifier_params:
classifier_params["n_jobs"] = int(classifier_params["n_jobs"])
if "n_estimators" in classifier_params:
classifier_params["n_estimators"] = int(
classifier_params["n_estimators"])
### PCA ###
if "_pca" in classifier_params:
del classifier_params["_pca"]
del classifier_params["_pca_dimensions"]
### GPC ###
if classifier_params["function"] == "gp":
if classifier_params["kernel"] == "constant":
classifier_params["kernel"] = ConstantKernel()
elif classifier_params["kernel"] == "rbf":
classifier_params["kernel"] = RBF()
elif classifier_params["kernel"] == "matern":
classifier_params["kernel"] = Matern(
nu=classifier_params["nu"])
del classifier_params["nu"]
elif classifier_params["kernel"] == "rational_quadratic":
classifier_params["kernel"] = RationalQuadratic(
length_scale=classifier_params["length_scale"],
alpha=classifier_params["alpha"])
del classifier_params["length_scale"]
del classifier_params["alpha"]
elif classifier_params["kernel"] == "exp_sine_squared":
classifier_params["kernel"] = ExpSineSquared(
length_scale=classifier_params["length_scale"],
periodicity=classifier_params["periodicity"])
del classifier_params["length_scale"]
del classifier_params["periodicity"]
### MLP ###
if classifier_params["function"] == "mlp":
classifier_params["hidden_layer_sizes"] = []
# set layer topology
if int(classifier_params["num_hidden_layers"]) == 1:
classifier_params["hidden_layer_sizes"].append(
classifier_params["hidden_size_layer1"])
del classifier_params["hidden_size_layer1"]
elif int(classifier_params["num_hidden_layers"]) == 2:
classifier_params["hidden_layer_sizes"].append(
classifier_params["hidden_size_layer1"])
classifier_params["hidden_layer_sizes"].append(
classifier_params["hidden_size_layer2"])
del classifier_params["hidden_size_layer1"]
del classifier_params["hidden_size_layer2"]
elif int(classifier_params["num_hidden_layers"]) == 3:
classifier_params["hidden_layer_sizes"].append(
classifier_params["hidden_size_layer1"])
classifier_params["hidden_layer_sizes"].append(
classifier_params["hidden_size_layer2"])
classifier_params["hidden_layer_sizes"].append(
classifier_params["hidden_size_layer3"])
del classifier_params["hidden_size_layer1"]
del classifier_params["hidden_size_layer2"]
del classifier_params["hidden_size_layer3"]
classifier_params["hidden_layer_sizes"] = [
int(x) for x in classifier_params["hidden_layer_sizes"]
] # convert to ints
# delete our fabricated keys
del classifier_params["num_hidden_layers"]
# print "Added stuff for DBNs! %s" % classifier_params
### DBN ###
if classifier_params["function"] == "dbn":
# print "Adding stuff for DBNs! %s" % classifier_params
classifier_params["layer_sizes"] = [
classifier_params["inlayer_size"]
]
# set layer topology
if int(classifier_params["num_hidden_layers"]) == 1:
classifier_params["layer_sizes"].append(
classifier_params["hidden_size_layer1"])
del classifier_params["hidden_size_layer1"]
elif int(classifier_params["num_hidden_layers"]) == 2:
classifier_params["layer_sizes"].append(
classifier_params["hidden_size_layer1"])
classifier_params["layer_sizes"].append(
classifier_params["hidden_size_layer2"])
del classifier_params["hidden_size_layer1"]
del classifier_params["hidden_size_layer2"]
elif int(classifier_params["num_hidden_layers"]) == 3:
classifier_params["layer_sizes"].append(
classifier_params["hidden_size_layer1"])
classifier_params["layer_sizes"].append(
classifier_params["hidden_size_layer2"])
classifier_params["layer_sizes"].append(
classifier_params["hidden_size_layer3"])
del classifier_params["hidden_size_layer1"]
del classifier_params["hidden_size_layer2"]
del classifier_params["hidden_size_layer3"]
classifier_params["layer_sizes"].append(
classifier_params["outlayer_size"])
classifier_params["layer_sizes"] = [
int(x) for x in classifier_params["layer_sizes"]
] # convert to ints
# set activation function
if classifier_params["output_act_funct"] == "Linear":
classifier_params["output_act_funct"] = Linear()
elif classifier_params["output_act_funct"] == "Sigmoid":
classifier_params["output_act_funct"] = Sigmoid()
elif classifier_params["output_act_funct"] == "Softmax":
classifier_params["output_act_funct"] = Softmax()
elif classifier_params["output_act_funct"] == "tanh":
classifier_params["output_act_funct"] = Tanh()
classifier_params["epochs"] = int(classifier_params["epochs"])
# delete our fabricated keys
del classifier_params["num_hidden_layers"]
del classifier_params["inlayer_size"]
del classifier_params["outlayer_size"]
# remove function key and return
del classifier_params["function"]
return classifier_params
for x0 in np.random.uniform(bounds[:, 0],
bounds[:, 1],
size=(n_restarts, dim)):
res = minimize(min_obj, x0=x0, bounds=bounds, method='L-BFGS-B')
if res.fun < min_val:
min_val = res.fun[0]
min_x = res.x
return min_x.reshape(-1, 1)
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import ConstantKernel, Matern
# Gaussian process with Mat??rn kernel as surrogate model
m52 = ConstantKernel(1.0) * Matern(length_scale=1.0, nu=2.5)
gpr = GaussianProcessRegressor(kernel=m52, alpha=noise**2)
# Initialize samples
X_sample = X_init
Y_sample = Y_init
# Number of iterations
n_iter = 30
plt.figure(figsize=(12, n_iter * 3))
plt.subplots_adjust(hspace=0.4)
for i in range(n_iter):
# Update Gaussian process with existing samples
gpr.fit(X_sample, Y_sample)
n_train = 3000
n_test = 10000
opt = Adam(1e-3)
batch_size = 500
gp = NNRegressor(layers,
opt=opt,
batch_size=batch_size,
maxiter=500,
gp=True,
verbose=True)
gp.fit(x_train, y_train)
#Can extract mapping z(x) and hyperparams for use in other learning algorithm
alph = gp.layers[-1].s_alpha
var = gp.layers[-1].var
A_full = gp.fast_forward(x_train)
kernel = ConstantKernel(var) * RBF(np.ones(1)) + WhiteKernel(alph)
A_test = gp.fast_forward(x_test[0:n_test])
yp = np.zeros(n_test)
std = np.zeros(n_test)
gp1 = GaussianProcessRegressor(kernel, optimizer=None)
gp1.fit(A_full[0:500], y_train[0:500])
mu, stdt = gp1.predict(A_test, return_std=True)
print("GP Regression:")
print(np.sqrt(np.mean((np.rint(mu) - y_test)**2)))
def interpolationLearner(xtrain, ytrain, xtest):
estimators = []
#model1
param_grid = {"alpha": [5e-05], "tol": [0.0001, 0.00001]}
model1 = GridSearchCV(Lasso(),
cv=4,
param_grid=param_grid,
scoring="neg_mean_squared_error")
estimators.append(
('Lasso',
model1)) # https://www.kaggle.com/apapiu/regularized-linear-models
#model2
param_grid = {
"alpha": [5e-05, 0.0005],
"l1_ratio": [0.9, 0.8],
"random_state": [3, 4]
}
model2 = GridSearchCV(
ElasticNet(),
cv=4,
param_grid=param_grid,
scoring="neg_mean_squared_error"
) # http://scikit-learn.org/stable/auto_examples/gaussian_process/plot_compare_gpr_krr.html#sphx-glr-auto-examples-gaussian-process-plot-compare-gpr-krr-py
estimators.append(('ENet', model2))
#model3
param_grid = {
"alpha": [0.6, 0.5],
"kernel": [ConstantKernel(1.0, (1e-3, 1e3)) + RBF()]
}
model3 = GridSearchCV(
KernelRidge(),
cv=4,
param_grid=param_grid,
scoring="neg_mean_squared_error"
) # http://scikit-learn.org/stable/auto_examples/gaussian_process/plot_compare_gpr_krr.html#sphx-glr-auto-examples-gaussian-process-plot-compare-gpr-krr-py
estimators.append(('KRR', model3))
#model4
param_grid = {
"loss": ["huber"],
"max_features": ["sqrt"],
"max_depth": [4, 5, 6]
}
model4 = GridSearchCV(
GradientBoostingRegressor(learning_rate=0.05,
max_depth=3,
n_estimators=2200),
cv=4,
param_grid=param_grid,
scoring="neg_mean_squared_error"
) # http://scikit-learn.org/stable/auto_examples/gaussian_process/plot_compare_gpr_krr.html#sphx-glr-auto-examples-gaussian-process-plot-compare-gpr-krr-py
estimators.append(('GBoost', model4))
learners = []
# Hagamos fit
for learner in np.arange(len(estimators)):
stime = time.time()
learners.append(estimators[learner][1].fit(xtrain, ytrain))
print("For " + str(estimators[learner][0]).upper())
print("")
print("Time for learner " + str(estimators[learner][0]) + " " +
str(time.time() - stime))
print("El learner " + str(estimators[learner][0]) +
" tiene un score de " + str(learners[learner].best_score_))
print("Los parametros para el learner " + str(estimators[learner][0]) +
" fueron " + str(learners[learner].best_params_))
print("")
joblib.dump(model1, 'model_cache/int_lasso_cache.pkl')
joblib.dump(model2, 'model_cache/int_elastic_cache.pkl')
joblib.dump(model3, 'model_cache/int_kernel_cache.pkl')
joblib.dump(model4, 'model_cache/int_gboost_cache.pkl')
# CV score para la media de los learners
predictTraino = np.column_stack(
[learner.predict(xtrain) for learner in learners])
mean_predictTraino = np.column_stack([np.mean(i) for i in predictTraino])
print("")
print("El score final de toda la interpolación en traino es " +
str(mean_squared_error(ytrain, mean_predictTraino[0])))
print("")
predict = np.column_stack([learner.predict(xtest) for learner in learners])
mean_predict = np.column_stack([i.mean() for i in predict])
return mean_predict[0]
def __init__(self):
super(Selector, self).__init__()
kernel = ConstantKernel(1.0) * Matern(length_scale=1.0, nu=2.5)
# self.model = GaussianProcessRegressor(alpha=0.2 ** 2)
self.model = GaussianProcessRegressor(kernel, alpha=0.2**2)
self.name = "GP_opt"
def cross_validation(self, X, y, cv = 5, scoring = 'neg_mean_squared_error', scale = True, fit = True):
"""You can decide kernels by using cross validation. You don't have to do this if you've already decide which kernels do you use.
Parameters
----------
X : list or something
[description]
y : list or something
[description]
cv : int, optional
[description], by default 5
scoring : str, optional
[description], by default 'neg_mean_squared_error'
scale : bool, optional
[description], by default True
fit : bool, optional
[description], by default True
"""
# クラス変数化
self.X = np.array(X).reshape(-1, 1)
self.y = np.array(y).reshape(-1, 1)
cv = min(cv, self.y.shape[0])
# 目的変数を正規化
if scale:
scaled_y, self.scaler_y = self._scale(self.y)
else:
scaled_y = self.y
self.scaler_y = None
# カーネル
kernels = [ConstantKernel() * DotProduct() + WhiteKernel(),
ConstantKernel() * RBF() + WhiteKernel(),
ConstantKernel() * RBF() + WhiteKernel() + ConstantKernel() * DotProduct(),
ConstantKernel() * RBF(np.ones(self.X.shape[1])) + WhiteKernel(),
ConstantKernel() * RBF(np.ones(self.X.shape[1])) + WhiteKernel() + ConstantKernel() * DotProduct(),
ConstantKernel() * Matern(nu=1.5) + WhiteKernel(),
ConstantKernel() * Matern(nu=1.5) + WhiteKernel() + ConstantKernel() * DotProduct(),
ConstantKernel() * Matern(nu=0.5) + WhiteKernel(),
ConstantKernel() * Matern(nu=0.5) + WhiteKernel() + ConstantKernel() * DotProduct(),
ConstantKernel() * Matern(nu=2.5) + WhiteKernel(),
ConstantKernel() * Matern(nu=2.5) + WhiteKernel() + ConstantKernel() * DotProduct()
]
params = {
'kernel':kernels
}
# GPRのインスタンス生成
gpr = GaussianProcessRegressor(alpha=0)
# kernel決定のためにcv
gscv = GridSearchCV(gpr, params, cv = cv, scoring = scoring)
gscv.fit(self.X, scaled_y)
# 後でアクセスできるように
self.best_score_ = gscv.best_score_
self.results = pd.DataFrame.from_dict(gscv.cv_results_)
# 最適なカーネルを使用
self.best_kernel_ = gscv.best_params_['kernel']
self.best_estimator = GaussianProcessRegressor(kernel = self.best_kernel_, alpha = 0)
if fit:
# fitさせる
self.best_estimator.fit(self.X, scaled_y)
max_sigma = [0] * total_increments #* max_n
max_error = [0] * total_increments #* max_n
print("bruh")
for index in range(1, len(
x_axis_array)): # max_n + 1): # i represents the current value of n
i = x_axis_array[index]
print("Finding every " + str(i) + "th element")
start_time = time.time()
X_train = normalize(X_train_original[::i])
X_test = normalize(X_test_original[::i])
y_train = normalize(y_train_original[::i])
y_test = normalize(y_test_original[::i])
kernel = ConstantKernel(1.0, (1e-3, 1e3)) * RBF([5] * X_train.shape[1],
(1e-2, 1e2))
gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=15)
gp.fit(X_train, y_train)
y_pred, stdev = gp.predict(X_test, return_std=True)
end_time = time.time()
print("done training")
time_array[index] = end_time - start_time
num_data_points[index] = len(y_train)
#maximum sigma
max_sigma[index] = max(stdev)
#maximum relative error
relative_error = (y_pred - y_test) / (y_test)
max_error[index] = max(relative_error)
def bayesianoptimization(X,
y,
candidates_of_X,
acquisition_function_flag,
cumulative_variance=None):
"""
Bayesian optimization
Gaussian process regression model is constructed between X and y.
A candidate of X with the highest acquisition function is selected using the model from candidates of X.
Parameters
----------
X: numpy.array or pandas.DataFrame
m x n matrix of X-variables of training dataset (m is the number of samples and n is the number of X-variables)
y: numpy.array or pandas.DataFrame
m x 1 vector of a y-variable of training dataset
candidates_of_X: numpy.array or pandas.DataFrame
Candidates of X
acquisition_function_flag: int
1: Mutual information (MI), 2: Expected improvement(EI),
3: Probability of improvement (PI) [0: Estimated y-values]
cumulative_variance: numpy.array or pandas.DataFrame
cumulative variance in mutual information (MI)[acquisition_function_flag=1]
Returns
-------
selected_candidate_number : int
selected number of candidates_of_X
selected_X_candidate : numpy.array
selected X candidate
cumulative_variance: numpy.array
cumulative variance in mutual information (MI)[acquisition_function_flag=1]
"""
X = np.array(X)
y = np.array(y)
if cumulative_variance is None:
cumulative_variance = np.empty(len(y))
else:
cumulative_variance = np.array(cumulative_variance)
relaxation_value = 0.01
delta = 10**-6
alpha = np.log(2 / delta)
autoscaled_X = (X - X.mean(axis=0)) / X.std(axis=0, ddof=1)
autoscaled_candidates_of_X = (candidates_of_X - X.mean(axis=0)) / X.std(
axis=0, ddof=1)
autoscaled_y = (y - y.mean(axis=0)) / y.std(axis=0, ddof=1)
gaussian_process_model = GaussianProcessRegressor(
ConstantKernel() * RBF() + WhiteKernel(), alpha=0)
gaussian_process_model.fit(autoscaled_X, autoscaled_y)
autoscaled_estimated_y_test, autoscaled_std_of_estimated_y_test = gaussian_process_model.predict(
autoscaled_candidates_of_X, return_std=True)
if acquisition_function_flag == 1:
acquisition_function_values = autoscaled_estimated_y_test + alpha**0.5 * (
(autoscaled_std_of_estimated_y_test**2 + cumulative_variance)**0.5
- cumulative_variance**0.5)
cumulative_variance = cumulative_variance + autoscaled_std_of_estimated_y_test**2
elif acquisition_function_flag == 2:
acquisition_function_values = (autoscaled_estimated_y_test - max(autoscaled_y) - relaxation_value) * \
norm.cdf((autoscaled_estimated_y_test - max(autoscaled_y) - relaxation_value) /
autoscaled_std_of_estimated_y_test) + \
autoscaled_std_of_estimated_y_test * \
norm.pdf((autoscaled_estimated_y_test - max(autoscaled_y) - relaxation_value) /
autoscaled_std_of_estimated_y_test)
elif acquisition_function_flag == 3:
acquisition_function_values = norm.cdf(
(autoscaled_estimated_y_test - max(autoscaled_y) -
relaxation_value) / autoscaled_std_of_estimated_y_test)
elif acquisition_function_flag == 0:
acquisition_function_values = autoscaled_estimated_y_test
selected_candidate_number = np.where(
acquisition_function_values == max(acquisition_function_values))[0][0]
selected_X_candidate = candidates_of_X[selected_candidate_number, :]
return selected_candidate_number, selected_X_candidate, cumulative_variance
def make_plot(days_ago, dates, mag):
print('Making plot...')
time_span = np.max(dates) - np.min(dates)
min_plot = 0.0
max_plot = 2
x_days = -120
# Make daily bins
nights = np.arange(0, 120, 1)
daily_mags = []
errors = []
for night in nights:
selector = np.where((days_ago < night + 1) & (days_ago > night))
n_obs = np.size(mag[selector])
flux = biweight_location(mag[selector])
error = np.std(mag[selector]) / np.sqrt(n_obs)
if error > 0.75:
error = 0
daily_mags.append(flux)
errors.append(error)
print(night, flux, error, n_obs, np.std(mag[selector]))
nights_all = nights.copy()
daily_mags_all = daily_mags.copy()
errors_all = errors.copy()
lookback = np.arange(1, 20, 1)
for missing_days in lookback:
nights = nights_all.copy()[missing_days:]
daily_mags = daily_mags_all.copy()[missing_days:]
errors = errors_all.copy()[missing_days:]
plt.errorbar(-(nights + 0.5),
daily_mags,
yerr=errors,
fmt='.k',
alpha=0.5)
plt.xlabel('Days from today')
plt.ylabel('Visual magnitude')
mid = biweight_location(mag)
plt.ylim(min_plot, max_plot)
plt.xlim(-100, 100)
plt.gca().invert_yaxis()
date_text = datetime.datetime.now().strftime("%d %b %Y")
plt.text(95,
min_plot + 0.1,
'AAVSO visual (by-eye) daily bins',
ha='right')
plt.text(95,
min_plot + 0.2,
'Gaussian process regression, Matern 3/2 kernel',
ha='right')
plt.text(95,
min_plot + 0.3,
'@betelbot update ' + date_text,
ha='right')
use_days = 60 - missing_days
X = np.array(nights + 0.5)
X = X[:use_days]
y = np.array(daily_mags)
y = y[:use_days]
X, y = cleaned_array(X, y)
length_scale = 1
kernel = ConstantKernel() + Matern(
length_scale=length_scale, nu=3 / 2) + WhiteKernel(noise_level=1)
X = X.reshape(-1, 1)
gp = gaussian_process.GaussianProcessRegressor(kernel=kernel)
gp.fit(X, y)
GaussianProcessRegressor(alpha=1e-10,
copy_X_train=True,
kernel=1**2 +
Matern(length_scale=length_scale, nu=1.5) +
WhiteKernel(noise_level=1),
n_restarts_optimizer=0,
normalize_y=False,
optimizer='fmin_l_bfgs_b',
random_state=None)
x_pred = np.linspace(60, -120, 250).reshape(-1, 1)
y_pred, sigma = gp.predict(x_pred, return_std=True)
plt.plot(-x_pred, y_pred, linestyle='dashed', color='blue')
plt.fill_between(-x_pred.ravel(),
y_pred + sigma,
y_pred - sigma,
alpha=0.5)
idx = 20 - missing_days
if idx < 10:
filename = "0" + str(idx) + '.png'
else:
filename = str(idx) + '.png'
plt.savefig(filename, bbox_inches='tight', dpi=100)
print('Plot made', filename)
plt.clf()
def main(_):
mnist = input_data.read_data_sets('/tmp/data/', one_hot=True, seed=12345)
random_weight_vector = np.random.uniform(low=0.1,
high=1.9,
size=TRAIN_INPUT_SIZE)
x = tf.placeholder(tf.float32, shape=(None, INPUT_DIM), name='x')
y = tf.placeholder(tf.float32, shape=(None, OUTPUT_DIM), name='y')
weight = tf.placeholder(tf.float32,
shape=(None, OUTPUT_DIM),
name='weight')
parallel_alphas = tf.placeholder(tf.float32,
shape=(FLAGS.num_parallel_alphas,
OUTPUT_DIM),
name='parallel_alphas')
unstack_parallel_alphas = tf.unstack(parallel_alphas, axis=0)
parallel_logits = []
parallel_losses = []
parallel_optimizers = []
validation_metrics = []
test_metrics = []
all_test_metrics = []
with tf.variable_scope('classifier'):
for alpha_index in range(FLAGS.num_parallel_alphas):
logits = classifier(x)
alpha = tf.reshape(unstack_parallel_alphas[alpha_index],
shape=[OUTPUT_DIM, 1])
optimizer, loss = optimization(logits, y, weight, alpha,
LEARNING_RATE)
parallel_logits.append(logits)
parallel_losses.append(loss)
parallel_optimizers.append(optimizer)
init = tf.global_variables_initializer()
classifiers_init = tf.variables_initializer(
tf.global_variables(scope='classifier'))
with tf.Session() as sess:
sess.run(init)
# GetCandidatesAlpha (Algorithm 2 in paper)
sample_alphas = np.zeros(shape=(0, OUTPUT_DIM))
for alpha_batch_index in range(FLAGS.num_alpha_batches):
sess.run(classifiers_init)
if FLAGS.uniform_weights:
alpha_batch = np.zeros(shape=(FLAGS.num_parallel_alphas,
OUTPUT_DIM))
elif FLAGS.random_alpha or alpha_batch_index < 1:
alpha_batch = sample_from_ball(
size=(FLAGS.num_parallel_alphas, OUTPUT_DIM),
sampling_radius=FLAGS.sampling_radius)
sample_alphas = np.concatenate([sample_alphas, alpha_batch])
else:
# Use LCB to generate candidates.
alpha_batch = np.zeros(shape=(0, OUTPUT_DIM))
sample_metrics = validation_metrics[:]
for alpha_index in range(FLAGS.num_parallel_alphas):
kernel = RBF(length_scale=FLAGS.sampling_radius,
length_scale_bounds=(
FLAGS.sampling_radius * 1e-3,
FLAGS.sampling_radius *
1e3)) * ConstantKernel(1.0, (1e-3, 1e3))
gp = GaussianProcessRegressor(kernel=kernel,
alpha=1e-4).fit(
sample_alphas,
np.log1p(sample_metrics))
candidates = sample_from_ball((10000, OUTPUT_DIM),
FLAGS.sampling_radius)
metric_mles, metric_stds = gp.predict(candidates,
return_std=True)
metric_lcbs = np.maximum(
np.expm1(metric_mles - 1.0 * metric_stds), 0.0)
metric_lcbs += np.random.random(
size=metric_lcbs.shape) * 0.001 # break ties
best_index = np.argmin(metric_lcbs)
best_alpha = [candidates[best_index]]
best_alpha_metric_estimate = np.minimum(
np.expm1(metric_mles[best_index] +
1.0 * metric_stds[best_index]), 1.0)
alpha_batch = np.concatenate([alpha_batch, best_alpha])
sample_alphas = np.concatenate([sample_alphas, best_alpha])
sample_metrics.append(best_alpha_metric_estimate)
# Training classifiers
for step in range(TRAINING_STEPS):
batch_index = range(
step * BATCH_SIZE % TRAIN_INPUT_SIZE,
step * BATCH_SIZE % TRAIN_INPUT_SIZE + BATCH_SIZE)
(batch_x, batch_y) = mnist.train.next_batch(BATCH_SIZE,
shuffle=False)
batch_weight = [[random_weight_vector[i]] * OUTPUT_DIM
for i in batch_index]
_, _ = sess.run(
[parallel_optimizers, parallel_losses],
feed_dict={
x: batch_x,
y: batch_y,
weight: batch_weight,
parallel_alphas: alpha_batch,
})
parallel_validation_logits = sess.run(parallel_logits,
feed_dict={
x:
mnist.validation.images,
y:
mnist.validation.labels,
})
parallel_validation_metrics = [
metric(mnist.validation.labels,
validation_logits,
all_digits=False)
for validation_logits in parallel_validation_logits
]
validation_metrics.extend(parallel_validation_metrics)
parallel_test_logits = sess.run(parallel_logits,
feed_dict={
x: mnist.test.images,
y: mnist.test.labels,
})
parallel_test_metrics = [
metric(mnist.test.labels, test_logits, all_digits=False)
for test_logits in parallel_test_logits
]
test_metrics.extend(parallel_test_metrics)
parallel_all_test_metrics = [
metric(mnist.test.labels, test_logits, all_digits=True)
for test_logits in parallel_test_logits
]
all_test_metrics.extend(parallel_all_test_metrics)
best_observed_index = np.argmin(validation_metrics)
print('[metric] validation={}'.format(
validation_metrics[best_observed_index]))
print('[metric] test={}'.format(test_metrics[best_observed_index]))
for i in range(10):
print('[all test metrics] {}={}'.format(
i, all_test_metrics[best_observed_index][i]))
def run_model(X,Y,perc,nbre_fois,model_name,list_variable):
regRandomForestRegressor_m1,regRandomForestRegressor_m2,regRandomForestRegressor_m3,regRandomForestRegressor_m4,regRandomForestRegressor_m5=None,None,None,None,None
"ensemble des modeles crées"
memoire_score=dict()
if 'RandomForestRegressor_m' in model_name:
regRandomForestRegressor_m1=RandomForestRegressor(max_depth=2, random_state=0)
regRandomForestRegressor_m2=RandomForestRegressor(max_depth=7, random_state=0)
regRandomForestRegressor_m3=RandomForestRegressor(max_depth=12, random_state=0)
regRandomForestRegressor_m4=RandomForestRegressor(max_depth=14, random_state=0)
regRandomForestRegressor_m5=RandomForestRegressor(max_depth=18, random_state=0)
if 'treeDecision_regresion_m' in model_name:
regtreeDecision_regresion_m1=tree.DecisionTreeRegressor(max_depth=2)
regtreeDecision_regresion_m2=tree.DecisionTreeRegressor(max_depth=7)
regtreeDecision_regresion_m3=tree.DecisionTreeRegressor(max_depth=12)
regtreeDecision_regresion_m4=tree.DecisionTreeRegressor(max_depth=14)
regtreeDecision_regresion_m5=tree.DecisionTreeRegressor(max_depth=18)
if 'svm_m' in model_name:
regvm_m1=svm.SVR()
regvm_m2=svm.LinearSVR()
if 'linear_model_OLS' in model_name:
reglinearie = linear_model.LinearRegression()
if 'KNeighborsRegressor_m' in model_name:
regKNeighborsRegressor=KNeighborsRegressor(n_neighbors=2)
if 'linear_model_Ridge' in model_name:
reglinear_model_Ridge1 = linear_model.Ridge(alpha=0.1)
reglinear_model_Ridge2 = linear_model.Ridge(alpha=0.1)
reglinear_model_Ridge3 = linear_model.Ridge(alpha=0.1)
reglinear_model_Ridge4 = linear_model.Ridge(alpha=0.1)
reglinear_model_Ridge5 = linear_model.Ridge(alpha=0.5)
reglinear_model_Ridge6 = linear_model.Ridge(alpha=0.6)
reglinear_model_Ridge7 = linear_model.Ridge(alpha=0.7)
reglinear_model_Ridge8 = linear_model.Ridge(alpha=0.8)
reglinear_model_Ridge9 = linear_model.Ridge(alpha=0.9)
if 'BayesianRidge_m' in model_name:
regBayesianRidge_m_F=linear_model.BayesianRidge(compute_score=False)
regBayesianRidge_m_T=linear_model.BayesianRidge(compute_score=True)
if 'AdaBoostRegressor_m' in model_name:
regAdaBoostRegressor_m=AdaBoostRegressor(random_state=0, n_estimators=100)
if 'MLPregression' in model_name:
reqMLPregression1=MLPRegressor(activation="identity",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression2=MLPRegressor(activation="identity",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression3=MLPRegressor(activation="identity",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression4=MLPRegressor(activation="logistic",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression5=MLPRegressor(activation="logistic",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression6=MLPRegressor(activation="logistic",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression7=MLPRegressor(activation="tanh",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression8=MLPRegressor(activation="tanh",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression9=MLPRegressor(activation="tanh",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression10=MLPRegressor(activation="relu",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression11=MLPRegressor(activation="relu",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression12=MLPRegressor(activation="relu",solver="lbfgs", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression13=MLPRegressor(activation="identity",solver="sgd", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression14=MLPRegressor(activation="identity",solver="sgd", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression15=MLPRegressor(activation="identity",solver="sgd", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression16=MLPRegressor(activation="logistic",solver="sgd", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression17=MLPRegressor(activation="logistic",solver="sgd", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression18=MLPRegressor(activation="logistic",solver="sgd", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression19=MLPRegressor(activation="tanh",solver="sgd", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression20=MLPRegressor(activation="tanh",solver="sgd", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression21=MLPRegressor(activation="relu",solver="sgd", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression22=MLPRegressor(activation="relu",solver="sgd", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression23=MLPRegressor(activation="relu",solver="sgd", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression24=MLPRegressor(activation="relu",solver="sgd", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression25=MLPRegressor(activation="identity",solver="adam", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression26=MLPRegressor(activation="identity",solver="adam", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression27=MLPRegressor(activation="identity",solver="adam", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression28=MLPRegressor(activation="logistic",solver="adam", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression29=MLPRegressor(activation="logistic",solver="adam", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression30=MLPRegressor(activation="logistic",solver="adam", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression31=MLPRegressor(activation="tanh",solver="adam", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression32=MLPRegressor(activation="tanh",solver="adam", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression33=MLPRegressor(activation="tanh",solver="adam", alpha=1e-7,hidden_layer_sizes=(20,1))
reqMLPregression34=MLPRegressor(activation="relu",solver="adam", alpha=1e-7,hidden_layer_sizes=(100,10))
reqMLPregression35=MLPRegressor(activation="relu",solver="adam", alpha=1e-7,hidden_layer_sizes=(70,30))
reqMLPregression36=MLPRegressor(activation="relu",solver="adam", alpha=1e-7,hidden_layer_sizes=(20,1))
if 'GaussianProcessRegressor_m' in model_name:
kernels = [1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0)),
1.0 * RationalQuadratic(length_scale=1.0, alpha=0.1),
1.0 * ExpSineSquared(length_scale=1.0, periodicity=3.0,
length_scale_bounds=(0.1, 10.0),
periodicity_bounds=(1.0, 10.0)),
ConstantKernel(0.1, (0.01, 10.0))
* (DotProduct(sigma_0=1.0, sigma_0_bounds=(0.1, 10.0)) ** 2),
1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0),
nu=1.5),DotProduct() + WhiteKernel()]
gpr1 = GaussianProcessRegressor(kernel=kernels[0],random_state=0)
gpr2 = GaussianProcessRegressor(kernel=kernels[1],random_state=0)
gpr3 = GaussianProcessRegressor(kernel=kernels[2],random_state=0)
gpr4 = GaussianProcessRegressor(kernel=kernels[3],random_state=0)
gpr5 = GaussianProcessRegressor(kernel=kernels[4],random_state=0)
gpr6 = GaussianProcessRegressor(kernel=kernels[5],random_state=0)
if 'SGDRegressor_m' in model_name:
regSGDRegressor_m1=linear_model.SGDRegressor(loss="squared_loss",max_iter=1000, tol=0.001)
regSGDRegressor_m2=linear_model.SGDRegressor(loss="huber",max_iter=1000, tol=0.001)
regSGDRegressor_m3=linear_model.SGDRegressor(loss="epsilon_insensitive",max_iter=1000, tol=0.001)
regSGDRegressor_m4=linear_model.SGDRegressor(loss="squared_epsilon_insensitive",max_iter=1000, tol=0.001)
X=X[list(list_variable)]
for k in range(1,nbre_fois):
train_index=[]
test_index=sample(list(X.index),perc)
for k in list(X.index):
if k not in test_index:
train_index.append(k)
X_test=X.iloc[test_index] # Chargement du X test
y_test=Y.iloc[test_index] # Chargement du y test
X_train=X.iloc[train_index] # Chargement du X train
y_train=Y.iloc[train_index] # Chargement du y train
if 'RandomForestRegressor_m' in model_name:
# arbre de decision
RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m1)
RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m2)
RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m3)
RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m4)
RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m5)
if 'treeDecision_regresion_m' in model_name:
treeDecision_regresion_m(X_train, y_train,X_test, y_test,regtreeDecision_regresion_m1)
treeDecision_regresion_m(X_train, y_train,X_test, y_test,regtreeDecision_regresion_m2)
treeDecision_regresion_m(X_train, y_train,X_test, y_test,regtreeDecision_regresion_m3)
treeDecision_regresion_m(X_train, y_train,X_test, y_test,regtreeDecision_regresion_m4)
treeDecision_regresion_m(X_train, y_train,X_test, y_test,regtreeDecision_regresion_m5)
if 'MLPregression' in model_name:
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression1)
except:
print("Toto")
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression2)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression3)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression4)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression5)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression6)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression7)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression8)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression9)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression10)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression11)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression12)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression13)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression14)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression15)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression16)
except:
pass
try:
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression17)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression18)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression19)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression20)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression21)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression22)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression23)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression24)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression25)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression26)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression27)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression28)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression29)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression30)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression31)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression32)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression33)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression34)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression35)
MLPregression(X_train, y_train,X_test, y_test,reqMLPregression36)
except:
pass
if 'AdaBoostRegressor_m' in model_name:
try:
AdaBoostRegressor_m(X_train, y_train,X_test, y_test,regAdaBoostRegressor_m)
except:
pass
if 'KNeighborsRegressor_m' in model_name:
try:
KNeighborsRegressor_m(X_train, y_train,X_test, y_test,regKNeighborsRegressor)
except:
pass
"model lineaire"
if 'linear_model_OLS' in model_name:
try:
"Ordinary Least Squares"
linear_model_OLS(X_train, y_train,X_test, y_test,reglinearie)
except:
pass
if 'linear_model_Ridge' in model_name:
#for alph in [0.1,0.4,0.5,0.6,0.7,0.8]:
try:
"Ordinary Least Rigde"
linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge1)
except:
pass
try:
"Ordinary Least Rigde"
linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge2)
except:
pass
try:
"Ordinary Least Rigde"
linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge3)
except:
pass
try:
"Ordinary Least Rigde"
linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge4)
except:
pass
try:
"Ordinary Least Rigde"
linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge5)
except:
pass
try:
"Ordinary Least Rigde"
linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge6)
except:
pass
try:
"Ordinary Least Rigde"
linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge7)
except:
pass
try:
"Ordinary Least Rigde"
linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge8)
except:
pass
try:
"Ordinary Least Rigde"
linear_model_Ridge(X_train, y_train,X_test, y_test,reglinear_model_Ridge9)
except:
pass
if 'BayesianRidge_m' in model_name:
try:
BayesianRidge_m(X_train, y_train,X_test, y_test,regBayesianRidge_m_T)
except:
pass
try:
BayesianRidge_m(X_train, y_train,X_test, y_test,regBayesianRidge_m_F)
except:
pass
if 'SGDRegressor_m' in model_name:
#for statut in ["squared_loss","huber","epsilon_insensitive","squared_epsilon_insensitive"]:
try:
SGDRegressor_m(X_train, y_train,X_test, y_test,regSGDRegressor_m1)
except:
pass
try:
SGDRegressor_m(X_train, y_train,X_test, y_test,regSGDRegressor_m2)
except:
pass
try:
SGDRegressor_m(X_train, y_train,X_test, y_test,regSGDRegressor_m3)
except:
pass
try:
SGDRegressor_m(X_train, y_train,X_test, y_test,regSGDRegressor_m4)
except:
pass
print("tot")
if 'svm_m' in model_name:
#"discrimation"
try:
svm_m(X_train, y_train,X_test, y_test,regvm_m1)
except:
pass
try:
svm_m(X_train, y_train,X_test, y_test,regvm_m2)
except:
pass
if 'gaussian_process_m' in model_name:
#gaussian_process_m
try:
gaussian_process_m(X_train, y_train,X_test, y_test,gpr1)
except:
pass
try:
gaussian_process_m(X_train, y_train,X_test, y_test,gpr2)
except:
pass
try:
gaussian_process_m(X_train, y_train,X_test, y_test,gpr3)
except:
pass
try:
gaussian_process_m(X_train, y_train,X_test, y_test,gpr4)
except:
pass
try:
gaussian_process_m(X_train, y_train,X_test, y_test,gpr5)
except:
pass
try:
gaussian_process_m(X_train, y_train,X_test, y_test,gpr6)
except:
pass
# try:
# exec("RandomForestRegressor_m(X_train, y_train,X_test, y_test,regRandomForestRegressor_m"+str(maxdep)+")")
# except:
# pass
# #print(regRandomForestRegressor_m9)
# print("toto")
#
# clf_SVR = svm.SVR()
# clf_SVR.fit(X_train, y_train)
# score_SVR = clf_SVR.score(X_test, y_test)
# print(" SVM")
# print(score_SVR)
# y_pred=clf_SVR.predict(X_test)
# print("gini:"+str(calcul_gini(y_test,y_pred)))
# print("RMSE:"+str(mean_squared_error(y_test, y_pred)))
# except:
# pass
# try:
# clf_tree = tree.DecisionTreeRegressor()
# clf_tree.fit(X_train, y_train)
# score_tree = clf_tree.score(X_test, y_test)
# print(" arbre de decision")
# print(score_tree)
# y_pred=clf_tree.predict(X_test)
# print("gini:"+str(calcul_gini(y_test,y_pred)))
# print("RMSE:"+str(mean_squared_error(y_test, y_pred)))
# except:
# pass
#
# "Neural network models (supervised)"
#
#
# try:
# # version classification
# modelMLPC = MLPClassifier(solver='lbfgs', alpha=1e-7,hidden_layer_sizes=(10, 5))
# #to learn
# modelMLPC.fit(X_train, y_train)
# #to predict
# print("Neural network ")
# yTest_predicted =modelMLPC.predict(X_test)
# print(accuracy_score(y_test,yTest_predicted))
# except:
# pass
#modelMLPC = MLPClassifier(solver='lbfgs', alpha=1e-7,hidden_layer_sizes=(10, 5))
#to learn
#modelMLPC.fit(X_train, y_train)
#to predict
#yTest_predicted =modelMLPC.predict(X_test)
#print(accuracy_score(y_test,yTest_predicted))
#.MLPRegressor(hidden_layer_sizes=(100, ), activation='relu', solver='adam', alpha=0.0001, batch_size='auto', learning_rate='constant', learning_rate_init=0.001, power_t=0.5, max_iter=200, shuffle=True, random_state=None, tol=0.0001, verbose=False, warm_start=False, momentum=0.9, nesterovs_momentum=True, early_stopping=False, validation_fraction=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-08, n_iter_no_change=10, max_fun=15000)
#clf_SGDCl=SGDClassifier(loss="hinge", penalty="l2", max_iter=5)
#clf_SGDCl.fit(X_train, y_train)
#score_SGDCl = clf_SGDCl.score(X_test, y_test)
#print(" arbre de decision")
#print(score_SGDCl)
#y_pred=clf_SGDCl.predict(X_test)
#print("gini:"+str(calcul_gini(y_test,y_pred)))
#print("RMSE:"+str(mean_squared_error(y_test, y_pred)))
#def neural_network(XTrain, yTrain,XTest,yTest):
# model = MLPClassifier(solver='lbfgs', alpha=1e-7,hidden_layer_sizes=(10, 5))
# #to learn
# model.fit(XTrain, yTrain)
# #to predict
# yTest_predicted =model.predict(XTest)
# accura=accuracy_score(yTest,yTest_predicted)
# print("accuracy :"+str(accura))
#
# return accura,model
# print("accuracy :"+str(accura))
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, ConstantKernel
import numpy.random as rd
rd.seed(42)
kernel = ConstantKernel(1.0, (1e-3, 1e3)) * RBF(1, (1e-2, 1e2))
gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9)
def f(x):
return x[0] * x[1]
n = 20
X = [[rd.random(), rd.random()] for _ in range(n)]
Y = [f(x) for x in X]
gp.fit(X, Y)
x = [[0.2, 0.2]]
print(gp.predict(x, return_std=True))
print(gp.predict(x, return_cov=True))
def log_marginal_likelihood(self, theta=None, eval_gradient=False):
"""Returns log-marginal likelihood of theta for training data.
Parameters
----------
theta : array-like, shape = (n_kernel_params,) or None
Kernel hyperparameters for which the log-marginal likelihood is
evaluated. If None, the precomputed log_marginal_likelihood
of ``self.kernel_.theta`` is returned.
eval_gradient : bool, default: False
If True, the gradient of the log-marginal likelihood with respect
to the kernel hyperparameters at position theta is returned
additionally. If True, theta must not be None.
Returns
-------
log_likelihood : float
Log-marginal likelihood of theta for training data.
log_likelihood_gradient : array, shape = (n_kernel_params,), optional
Gradient of the log-marginal likelihood with respect to the kernel
hyperparameters at position theta.
Only returned when eval_gradient is True.
"""
if theta is None:
if eval_gradient:
raise ValueError(
"Gradient can only be evaluated for theta!=None")
return self.log_marginal_likelihood_value_
kernel = self.kernel_.clone_with_theta(theta)
if eval_gradient:
K, K_gradient = kernel(self.X_train_, eval_gradient=True)
else:
K = kernel(self.X_train_)
# Compute log-marginal-likelihood Z and also store some temporaries
# which can be reused for computing Z's gradient
Z, (pi, W_sr, L, b, a) = \
self._posterior_mode(K, return_temporaries=True)
if not eval_gradient:
return Z
# Compute gradient based on Algorithm 5.1 of GPML
d_Z = np.empty(theta.shape[0])
# XXX: Get rid of the np.diag() in the next line
R = W_sr[:, np.newaxis] * cho_solve((L, True), np.diag(W_sr)) # Line 7
C = solve(L, W_sr[:, np.newaxis] * K) # Line 8
# Line 9: (use einsum to compute np.diag(C.T.dot(C))))
s_2 = -0.5 * (np.diag(K) - np.einsum('ij, ij -> j', C, C)) \
* (pi * (1 - pi) * (1 - 2 * pi)) # third derivative
for j in range(d_Z.shape[0]):
C = K_gradient[:, :, j] # Line 11
# Line 12: (R.T.ravel().dot(C.ravel()) = np.trace(R.dot(C)))
s_1 = .5 * a.T.dot(C).dot(a) - .5 * R.T.ravel().dot(C.ravel())
b = C.dot(self.y_train_ - pi) # Line 13
s_3 = b - K.dot(R.dot(b)) # Line 14
d_Z[j] = s_1 + s_2.T.dot(s_3) # Line 15
return Z, d_Z
def predict_iterative(pickle_prefix, fout):
kernel = ConstantKernel() + Matern(length_scale=2,
nu=3 / 2) + WhiteKernel(noise_level=1)
gp = gaussian_process.GaussianProcessRegressor(kernel=kernel)
probabilities_pickle_file = pickle_prefix + "probabilities.pickle"
selects_pickle_file = pickle_prefix + "selects.pickle"
ys_by_image_id = load_data(probabilities_pickle_file)
selects_by_image_id = load_data(selects_pickle_file)
max_ids = 5
num_initial_training = 200
mses = []
num_predictions = 0.
num_correct = 0.
num_positives = 0.
num_negatives = 0.
false_positives = 0.
false_negatives = 0.
for i, (image_id, ys) in enumerate(ys_by_image_id.iteritems()):
if i > max_ids:
break
ys_train = ys[:num_initial_training]
X = np.array(range(len(ys_train))).reshape(-1, 1)
gp.fit(X, ys_train)
selects = selects_by_image_id[image_id]
print("-------------image-{}------------------".format(image_id))
print(
"Fit image-{} with {} samples with {} average probability".format(
image_id, len(ys_train), round(np.mean(ys_train), 4)))
for i in range(num_initial_training + 1, len(ys)):
is_selected_actual = 1 - selects[i]
y_actual = ys[i]
xs_test = np.array([i]).reshape(-1, 1)
coin_flip = np.random.uniform(0, 1)
if coin_flip < y_actual:
is_selected_hypothetical = 1
else:
is_selected_hypothetical = 0
y_pred, std = gp.predict(xs_test, return_std=True)
is_selected_pred = predict_selection(coin_flip, y_pred, std)
fout.write("prediction,{},{},{},{},{},{},{}\n".format(
image_id, (xs_test[0]).tolist()[0], y_pred[0], y_actual,
is_selected_actual, is_selected_hypothetical,
is_selected_pred))
fout.flush()
# Calculate accuracy of regression
mse = get_mse(y_pred, [ys[i]])
mses.append(mse)
#print("MSE: {0:5f}".format(mse))
# Calculate accuracy of selection
num_predictions += 1
if is_selected_pred == is_selected_actual:
num_correct += 0
if is_selected_actual:
num_positives += 1
if not is_selected_pred:
false_negatives += 1
else:
num_negatives += 1
if not is_selected_pred:
false_positives += 1
ys_train = ys[:i]
X = np.array(range(len(ys_train))).reshape(-1, 1)
gp.fit(X, ys_train)
accuracy = num_correct / num_predictions
false_positive_rate = false_positives / num_negatives
false_negative_rate = false_negatives / num_positives
print("Intermediate Acc:{}, FPR:{}, FNR:{}".format(
accuracy, false_positive_rate, false_negative_rate))
accuracy = num_correct / num_predictions
false_positive_rate = false_positives / num_negatives
false_negative_rate = false_negatives / num_positives
return accuracy, false_positive_rate, false_negative_rate
def plot():
i=-1#for i in range(2, 9):
Xgps = Data.Xgps
Ygps = Data.Ygps
"""plot Google Maps with GPS"""
gmap = gmplot.GoogleMapPlotter(
sum(Ygps[:,0])/float(len(Ygps[:,0])),
sum(Ygps[:,1])/float(len(Ygps[:,1])), 18,
'AIzaSyC2I6z5RX44ZDn5z1-PiVFoEIIEVp5scKI')
""" plot GPS-GPR-Prediction"""
#add smoothed gps data
x = np.atleast_2d(np.linspace(min(Xgps), max(Xgps), 1000)).T
#RBF is a radial kernel with a length scale parameter controlling the distance
# of two points influencing each other (correlation).
#ConstantKernel defines a constant.
#DotProduct is a linear kernel with a noise level parameter.
kernel = RBF(0.01, (1e-3, 3)) *\
ConstantKernel(0.001, (1e-3, 10)) +\
WhiteKernel(1e-7, (1e-13, 1e-9))# +\
#ConstantKernel(1, (1, 10000)) *\
#DotProduct(0, (0, 180))
#noise = 0.0001
gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=3, normalize_y=True)
gp.fit(Xgps, Ygps)
y_pred, sigma = gp.predict(x, return_std=True)
#plot some samples from posterior
sample = gp.sample_y(x, n_samples=10)
for s in range(10):
gmap.plot(sample[:,0,s], sample[:,1,s], 'orange', edge_width=1)
#plot gps data
gmap.plot(Ygps[:,0], Ygps[:,1], 'cornflowerblue', edge_width=3)
gmap.scatter(Ygps[:,0], Ygps[:,1], 'blue', marker=False, size=1)
#plot mean posterior
gmap.plot(y_pred[:,0], y_pred[:,1], 'red', edge_width=3)#, label=u'Prediction')
fig, ax = plt.subplots()
ax.plot(x, y_pred[:,0], 'r-', label='mean posterior')
ax.plot(Xgps, Ygps[:,0], 'bo', label='original data')
ax.fill(np.concatenate([x, x[::-1]]),
np.concatenate([y_pred[:,0] - 1.9600 * sigma,
(y_pred[:,0] + 1.9600 * sigma)[::-1]]),
alpha=.3, fc='b', ec='None', label='95% confidence interval')
fig.show()
#==============================================================================
# gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=3, normalize_y=False)
# mean = Ygps.mean(0)
# std = (Ygps-mean).std(0)
# gp.fit(Xgps, (Ygps-mean)/std)
# y_pred, sigma = gp.predict(x, return_std=True)
# #plot mean posterior
# gmap.plot((y_pred[:,0]*std[0]+mean[0]), (y_pred[:,1]*std[1]+mean[1]), 'red', edge_width=1)#, label=u'Prediction')
# #plot some samples from posterior
# sample = gp.sample_y(x, n_samples=10)
# for s in range(10):
# gmap.plot((sample[:,0,s]*std[0]+mean[0]), (sample[:,1,s]*std[1]+mean[1]), 'orange', edge_width=1)
#
#==============================================================================
#gmap.polygon(np.concatenate((Ygps[:,0]-0.001, Ygps[:,0]+0.001)), np.concatenate((Ygps[:,1]+0.001,Ygps[:,1]-0.001)))
"""draw it and change to sattelite view"""
fileName = "googlemapsplot"+str(i)+".html"
print("writing result to "+fileName)
gmap.draw(fileName)
with open(fileName, 'r') as f:
s = f.read()
with open(fileName, 'w') as f:
s = s.replace("MapTypeId.ROADMAP", "MapTypeId.SATELLITE")
f.write(s)
def __init__(self, config, f, save=None, status=None):
self.check_config(config)
self.config = config
self.save = save
local_config = {}
# self.n_jobs = self.config.n_jobs
if self.config.old_type:
self.pnames = self.config.pnames
self.msteps = self.config.msteps
self.base = self.config.pinits
self.in_real = self.config.in_real
# All dimensions are integers in our case, but we can optimize in real (more noise)
# Here we will consider the scale as the +/- range from the initial value
dimensions = []
x0 = []
# Dimensions will be automatically normalized by skopt
if type(self.config.pscale) == list:
for pi, si in zip(self.config.pinits, self.config.pscale):
x0.append(pi)
start = pi - si
end = pi + si
if self.in_real:
dimensions.append(Real(start, end))
else:
dimensions.append(Integer(start, end))
else:
for pi, si, ei in zip(self.config.pinits, self.config.pmin,
self.config.pmax):
x0.append(pi)
if self.in_real:
dimensions.append(Real(si, ei))
else:
dimensions.append(Integer(si, ei))
regressor = self.config.regressor
normalize_y = self.config.normalize
fix_noise = self.config.fix_noise
games = self.config.games
elo = self.config.elo
isotropic = self.config.isotropic
nu = self.config.nu
ropoints = self.config.ropoints
acq_func = self.config.acq_func
simul = self.config.simul
else:
self.pnames = list(
map(lambda p: p.name, self.config.optimization.params))
self.msteps = self.config.optimization.msteps
self.base = list(
map(lambda p: p.ini, self.config.optimization.params))
self.in_real = self.config.optimization.in_real
# All dimensions are integers in our case, but we can optimize in real (more noise)
# Here we will consider the scale as the +/- range from the initial value
dimensions = []
x0 = []
# Dimensions will be automatically normalized by skopt
for param in self.config.optimization.params:
x0.append(param.ini)
if self.in_real:
dimensions.append(Real(param.min, param.max))
else:
dimensions.append(Integer(param.min, param.max))
regressor = self.config.method.params.regressor
normalize_y = self.config.method.params.normalize
fix_noise = self.config.method.params.fix_noise
games = self.config.eval.params.games
elo = self.config.eval.params.elo
isotropic = self.config.method.params.isotropic
nu = self.config.method.params.nu
ropoints = self.config.method.params.ropoints
isteps = self.config.method.params.isteps
acq_func = self.config.method.params.acq_func
simul = self.config.eval.type == 'simul'
self.is_gp = False
if regressor == 'GP':
self.is_gp = True
# Y normalization: GP assumes mean 0, or otherwise normalize (which seems not to work well,
# as the sample mean should be taken only for random points - we could calculate the mean
# ourselves from the initial random points - not done for now)
# Yet is normalization the way to go...
# The noise level is fix in our case, as we play a fixed number of games per step
# (exception: initial / reference point, with noise = 0 - we ignore this)
# It depends of number of games per step and the loss function (elo/elowish)
# The formula below is an ELO error approximation for the confidence interval of 95%,
# which lies by about 2 sigma - we can compute sigma of the error
if fix_noise:
noise_level_bounds = 'fixed'
sigma = 250. / math.sqrt(games)
if not elo:
sigma = sigma * math.log(10) / 400.
noise_level = sigma * sigma
else:
if elo and not normalize_y:
noise_level_bounds = (1e-2, 1e4)
noise_level = 10
else:
noise_level_bounds = (1e-6, 1e0)
noise_level = 0.01
# Length scales: because skopt normalizes the dimensions automatically, it is unclear
# how to proceed here, as the kernel does not know about that normalization...
length_scale_bounds = (1e-3, 1e4)
length_scale = 1
# Isotropic or anisotropic kernel
if not isotropic:
length_scale = length_scale * np.ones(len(dimensions))
# Matern kernel with configurable parameter nu (1.5 = once differentiable functions),
# white noise kernel and a constant kernel for mean estimatation
kernel = \
ConstantKernel() \
+ WhiteKernel(noise_level=noise_level, noise_level_bounds=noise_level_bounds) \
+ 1.0 * Matern(nu=nu, length_scale=length_scale, \
length_scale_bounds=length_scale_bounds)
# We put alpha=0 because we count in the kernel for the noise
# n_restarts_optimizer is important to find a good fit! (but it costs time)
rgr = GaussianProcessRegressor(kernel=kernel,
alpha=0.0,
normalize_y=normalize_y,
n_restarts_optimizer=ropoints)
else:
rgr = regressor
# When we start to use the regressor, we should have enough random points
# for a good space exploration
if isteps == 0:
n_initial_points = max(self.msteps // 10, len(dimensions) + 1)
else:
n_initial_points = isteps
self.optimizer = Optimizer(
dimensions=dimensions,
base_estimator=rgr,
acq_func=acq_func,
acq_optimizer='sampling', # without this I got this error:
# grad = self.kernel_.gradient_x(X[0], self.X_train_)
# AttributeError: 'Sum' object has no attribute 'gradient_x'
n_initial_points=n_initial_points,
initial_point_generator='lhs', # latin hypercube, just a try
n_jobs=-1, # use all cores in the estimator
model_queue_size=2)
self.done = False
if status is None:
# When we start, we know that the initial point is the reference, mark it with 0
# Unless we simulate!
if simul:
self.theta = None
self.best = None
self.xi = []
self.yi = []
else:
self.theta = x0
self.best = 0
self.xi = [x0]
self.yi = [0]
self.optimizer.tell(x0, 0)
else:
self.theta = status['theta']
self.best = status['best']
self.xi = status['xi']
self.yi = status['yi']
# Here: because we maximize (while the bayes optimizer minimizes), negate!
self.optimizer.tell(self.xi, list(map(lambda y: -y, self.yi)))
# Because we added the reference result, we have now one measurement more than steps
self.step = len(self.xi)
self.func = f
y = pd.read_csv('weeks.csv', header=0)
df = df_1.transpose()
df = df[1:]
X = df
median_pred = pd.DataFrame({"patient_id": X.index})
y = pd.concat([median_pred, y], axis=1)
y['patient_id'] = y['patient_id'].astype(str)
Y = y.set_index('patient_id')
Y = Y["weeks"]
Y
X["p"] = [x.split("_")[0] for x in X.index]
y["p"] = [x.split("_")[0] for x in Y.index]
ker_rbf = ConstantKernel(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2))
ker_rq = ConstantKernel(1.0, (1e-3, 1e3)) * RationalQuadratic(alpha=0.1,
length_scale=1)
kernel_list = [ker_rbf, ker_rq]
param_grid = {
"kernel": kernel_list,
"alpha": [1e1],
"optimizer": ["fmin_l_bfgs_b"],
"n_restarts_optimizer": [10, 15, 20, 25]
}
prediction = []
i = 0
scaler = MinMaxScaler(feature_range=(0, 1))
sc = StandardScaler()
def __init__(self,
sample,
data,
kernel=None,
noise=False,
global_optimizer=True):
r"""Create the predictor.
Uses sample and data to construct a predictor using Gaussian Process.
Input is to be normalized before and depending on the number of
parameters, the kernel is adapted to be anisotropic.
:attr:`self.data` contains the predictors as a list(array) of the size
of the `ouput`. A predictor per line of `data` is created. This leads
to a line of predictors that predicts a new column of `data`.
If :attr:`noise` is a float, it will be used as :attr:`noise_level` by
:class:`sklearn.gaussian_process.kernels.WhiteKernel`. Otherwise, if
:attr:`noise` is ``True``, default values are use for the WhiteKernel.
If :attr:`noise` is ``False``, no noise is added.
A multiprocessing strategy is used:
1. Create a process per mode, do not create if only one,
2. Create `n_restart` (3 by default) processes by process.
In the end, there is :math:`N=n_{restart} \times n_{modes})` processes.
If there is not enought CPU, :math:`N=\frac{n_{cpu}}{n_{restart}}`.
:param array_like sample: Sample used to generate the data
(n_samples, n_features).
:param array_like data: Observed data (n_samples, n_features).
:param kernel: Kernel from scikit-learn.
:type kernel: :class:`sklearn.gaussian_process.kernels`.*.
:param float/bool noise: Noise used into kriging.
:param bool global_optimizer: Whether to do global optimization or
gradient based optimization to estimate hyperparameters.
"""
sample = np.atleast_2d(sample)
dim = sample.shape[1]
self.model_len = data.shape[1]
if self.model_len == 1:
data = data.ravel()
if kernel is not None:
self.kernel = kernel
else:
# Define the model settings
l_scale = (1.0, ) * dim
self.scale_bounds = [(0.01, 100)] * dim
self.kernel = ConstantKernel() * Matern(
length_scale=l_scale, length_scale_bounds=self.scale_bounds)
# Add a noise on the kernel using WhiteKernel
if noise:
if isinstance(noise, bool):
noise = WhiteKernel()
else:
noise = WhiteKernel(noise_level=noise)
self.kernel += noise
# Global optimization
args_optim = {'kernel': self.kernel, 'normalize_y': True}
if global_optimizer:
args_optim.update({
'optimizer': self._optim_evolution,
'n_restarts_optimizer': 0
})
self.n_restart = 3
else:
args_optim.update({'n_restarts_optimizer': 10 * dim})
self.n_restart = 1
# Define the CPU multi-threading/processing strategy
n_cpu_system = cpu_system()
self.n_cpu = self.n_restart * self.model_len
if (n_cpu_system // (self.n_restart * self.model_len) < 1)\
or (self.n_cpu > n_cpu_system // self.n_restart):
self.n_cpu = n_cpu_system // self.n_restart
self.n_cpu = 1 if self.n_cpu == 0 else self.n_cpu
def model_fitting(column):
"""Fit an instance of :class:`sklearn.GaussianProcessRegressor`."""
gp = GaussianProcessRegressor(**args_optim)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
data = gp.fit(sample, column)
hyperparameter = np.exp(gp.kernel_.theta)
# Convergence check with bounds only when kernel not user defined
if kernel is None:
hyper_bounds = all([
i[0] < j < i[1]
for i, j in zip(self.scale_bounds, hyperparameter[1:dim +
1])
])
if not hyper_bounds:
self.logger.warning("Hyperparameters optimization not "
"converged: {}".format(gp.kernel_))
return data, hyperparameter
# Create a predictor per data, parallelize if several data
if self.model_len > 1:
pool = NestedPool(self.n_cpu)
results = pool.imap(model_fitting, data.T)
results = list(results)
pool.terminate()
else:
results = [model_fitting(data)]
# Gather results
self.gp_models, self.hyperparameter = zip(*results)
self.logger.debug("Kernels:\n{}".format(
[gp.kernel_ for gp in self.gp_models]))
def log_marginal_likelihood(self, theta=None, eval_gradient=False):
"""Returns log-marginal likelihood of theta for training data.
Parameters
----------
theta : array-like, shape = (n_kernel_params,) or None
Kernel hyperparameters for which the log-marginal likelihood is
evaluated. If None, the precomputed log_marginal_likelihood
of ``self.kernel_.theta`` is returned.
eval_gradient : bool, default: False
If True, the gradient of the log-marginal likelihood with respect
to the kernel hyperparameters at position theta is returned
additionally. If True, theta must not be None.
Returns
-------
log_likelihood : float
Log-marginal likelihood of theta for training data.
log_likelihood_gradient : array, shape = (n_kernel_params,), optional
Gradient of the log-marginal likelihood with respect to the kernel
hyperparameters at position theta.
Only returned when eval_gradient is True.
"""
if theta is None:
if eval_gradient:
raise ValueError(
"Gradient can only be evaluated for theta!=None")
return self.log_marginal_likelihood_value_
kernel = self.kernel_.clone_with_theta(theta)
if eval_gradient:
K, K_gradient = kernel(self.X_train_, eval_gradient=True)
else:
K = kernel(self.X_train_)
# Compute log-marginal-likelihood Z and also store some temporaries
# which can be reused for computing Z's gradient
Z, (pi, W_sr, L, b, a) = \
self._posterior_mode(K, return_temporaries=True)
if not eval_gradient:
return Z
# Compute gradient based on Algorithm 5.1 of GPML
d_Z = np.empty(theta.shape[0])
# XXX: Get rid of the np.diag() in the next line
R = W_sr[:, np.newaxis] * cho_solve((L, True), np.diag(W_sr)) # Line 7
C = solve(L, W_sr[:, np.newaxis] * K) # Line 8
# Line 9: (use einsum to compute np.diag(C.T.dot(C))))
s_2 = -0.5 * (np.diag(K) - np.einsum('ij, ij -> j', C, C)) \
* (pi * (1 - pi) * (1 - 2 * pi)) # third derivative
for j in range(d_Z.shape[0]):
C = K_gradient[:, :, j] # Line 11
# Line 12: (R.T.ravel().dot(C.ravel()) = np.trace(R.dot(C)))
s_1 = .5 * a.T.dot(C).dot(a) - .5 * R.T.ravel().dot(C.ravel())
b = C.dot(self.y_train_ - pi) # Line 13
s_3 = b - K.dot(R.dot(b)) # Line 14
d_Z[j] = s_1 + s_2.T.dot(s_3) # Line 15
return Z, d_Z
def main(_):
num_steps_autoencoder = 0 if FLAGS.uniform_weights else TRAINING_STEPS
training_df = pd.read_csv(FLAGS.training_data_path, header=0, sep=',')
testing_df = pd.read_csv(FLAGS.testing_data_path, header=0, sep=',')
validation_df = pd.read_csv(FLAGS.validation_data_path, header=0, sep=',')
train_labels = training_df['label']
validation_labels = validation_df['label']
test_labels = testing_df['label']
train_population = training_df['population']
train_features = training_df[FEATURES]
validation_features = validation_df[FEATURES]
test_features = testing_df[FEATURES]
train_wqs = training_df['racePctWhite_quantile']
validation_wqs = validation_df['racePctWhite_quantile']
test_wqs = testing_df['racePctWhite_quantile']
tf.reset_default_graph()
x = tf.placeholder(tf.float32, shape=(None, len(FEATURES)), name='x')
y = tf.placeholder(tf.float32, shape=(None, OUTPUT_DIM), name='y')
population = tf.placeholder(tf.float32,
shape=(None, OUTPUT_DIM),
name='population')
xy = tf.concat([x, y], axis=1)
autoencoder_layer1 = tf.layers.dense(inputs=xy,
units=10,
activation=tf.sigmoid)
autoencoder_embedding_layer = tf.layers.dense(inputs=autoencoder_layer1,
units=EMBEDDING_DIM,
activation=tf.sigmoid)
autoencoder_layer3 = tf.layers.dense(inputs=autoencoder_embedding_layer,
units=10,
activation=tf.sigmoid)
autoencoder_out_x = tf.layers.dense(inputs=autoencoder_layer3,
units=len(FEATURES))
autoencoder_out_y_logits = tf.layers.dense(inputs=autoencoder_layer3,
units=OUTPUT_DIM)
autoencoder_y_loss = tf.losses.hinge_loss(labels=y,
logits=autoencoder_out_y_logits)
autoencoder_x_loss = tf.losses.mean_squared_error(
labels=x, predictions=autoencoder_out_x)
autoencoder_loss = autoencoder_x_loss + autoencoder_y_loss
autoencoder_optimizer = tf.train.AdamOptimizer(LEARNING_RATE).minimize(
autoencoder_loss)
parallel_logits = []
parallel_losses = []
parallel_optimizers = []
parallel_alphas = tf.placeholder(tf.float32,
shape=(NUM_PARALLEL_ALPHAS,
EMBEDDING_DIM),
name='parallel_alphas')
unstack_parallel_alphas = tf.unstack(parallel_alphas, axis=0)
embedding = tf.placeholder(tf.float32,
shape=(None, EMBEDDING_DIM),
name='embedding')
with tf.variable_scope('classifiers'):
for alpha_index in range(NUM_PARALLEL_ALPHAS):
logits = classifier(x)
alpha = tf.reshape(unstack_parallel_alphas[alpha_index],
shape=[EMBEDDING_DIM, 1])
optimizer, loss = optimization(logits, y, population, embedding,
alpha)
parallel_logits.append(logits)
parallel_losses.append(loss)
parallel_optimizers.append(optimizer)
init = tf.global_variables_initializer()
classifiers_init = tf.variables_initializer(
tf.global_variables(scope='classifiers'))
kernel = RBF(length_scale=FLAGS.sampling_radius,
length_scale_bounds=(FLAGS.sampling_radius * 1e-3,
FLAGS.sampling_radius *
1e3)) * ConstantKernel(1.0, (1e-3, 1e3))
alphas = np.zeros(shape=(0, EMBEDDING_DIM))
validation_metrics = []
test_metrics = []
with tf.Session() as sess:
sess.run(init)
# Training autoencoder
for _ in range(num_steps_autoencoder):
batch_index = random.sample(range(len(train_labels)), BATCH_SIZE)
batch_x = train_features.iloc[batch_index, :].values
batch_y = train_labels.iloc[batch_index].values.reshape(
BATCH_SIZE, 1)
_, _ = sess.run([autoencoder_optimizer, autoencoder_loss],
feed_dict={
x: batch_x,
y: batch_y,
})
# GetCandidatesAlpha (Algorithm 2 in paper)
for alpha_batch_index in range(NUM_ALPHA_BATCHES):
sess.run(classifiers_init)
if FLAGS.uniform_weights:
alpha_batch = np.zeros(shape=(NUM_PARALLEL_ALPHAS,
EMBEDDING_DIM))
elif alpha_batch_index == 0:
# We first start uniformly.
alpha_batch = sample_from_ball(
size=(NUM_PARALLEL_ALPHAS, EMBEDDING_DIM),
sampling_radius=FLAGS.sampling_radius)
else:
# Use UCB to generate candidates.
alpha_batch = np.zeros(shape=(0, EMBEDDING_DIM))
sample_alphas = np.copy(alphas)
sample_validation_metrics = [m[0] for m in validation_metrics]
candidates = sample_from_ball(
size=(10000, EMBEDDING_DIM),
sampling_radius=FLAGS.sampling_radius)
for alpha_index in range(NUM_PARALLEL_ALPHAS):
gp = GaussianProcessRegressor(
kernel=kernel,
alpha=1e-1).fit(sample_alphas,
sample_validation_metrics)
metric_mles, metric_stds = gp.predict(candidates,
return_std=True)
metric_lcbs = metric_mles - 1.0 * metric_stds
best_index = np.argmin(metric_lcbs)
best_alpha = [candidates[best_index]]
best_alpha_metric_ucb = metric_mles[best_index] \
+ 1.0 * metric_stds[best_index]
alpha_batch = np.concatenate([alpha_batch, best_alpha])
# Add candidate to the GP, assuming the metric observation is the LCB.
sample_alphas = np.concatenate([sample_alphas, best_alpha])
sample_validation_metrics.append(best_alpha_metric_ucb)
# Training classifiers
for _ in range(TRAINING_STEPS):
batch_index = random.sample(range(len(train_labels)),
BATCH_SIZE)
batch_x = train_features.iloc[batch_index, :].values
batch_y = train_labels.iloc[batch_index].values.reshape(
BATCH_SIZE, 1)
batch_population = train_population.iloc[
batch_index].values.reshape(BATCH_SIZE, 1)
batch_embedding = sess.run(autoencoder_embedding_layer,
feed_dict={
x: batch_x,
y: batch_y,
})
_, _ = sess.run(
[parallel_optimizers, parallel_losses],
feed_dict={
x: batch_x,
y: batch_y,
population: batch_population,
embedding: batch_embedding,
parallel_alphas: alpha_batch,
})
parallel_train_logits = sess.run(parallel_logits,
feed_dict={
x:
train_features.values,
y:
train_labels.values.reshape(
len(train_labels), 1),
})
alphas = np.concatenate([alphas, alpha_batch])
parallel_validation_logits = sess.run(
parallel_logits,
feed_dict={
x: validation_features.values,
y:
validation_labels.values.reshape(len(validation_labels),
1),
})
parallel_test_logits = sess.run(parallel_logits,
feed_dict={
x:
test_features.values,
y:
test_labels.values.reshape(
len(test_labels), 1),
})
parallel_thresholds = [
find_threshold(train_labels, train_logits, train_wqs,
FLAGS.post_shift)
for train_logits in parallel_train_logits
]
logits_thresholds = zip(parallel_validation_logits,
parallel_thresholds)
parallel_validation_metrics = [
metrics(validation_labels, logits, validation_wqs, thresholds)
for (logits, thresholds) in logits_thresholds
]
validation_metrics.extend(parallel_validation_metrics)
parallel_test_metrics = [
metrics(test_labels, test_logits, test_wqs, thresholds)
for (test_logits, thresholds
) in zip(parallel_test_logits, parallel_thresholds)
]
test_metrics.extend(parallel_test_metrics)
best_observed_index = np.argmin([m[0] for m in validation_metrics])
print('[metric] validation_acc={}'.format(
validation_metrics[best_observed_index][0]))
print('[metric] validation_violation={}'.format(
validation_metrics[best_observed_index][1]))
print('[metric] test_acc={}'.format(test_metrics[best_observed_index][0]))
print('[metric] test_violation={}'.format(
test_metrics[best_observed_index][1]))
return 0
