def test_lml_improving():
""" Test that hyperparameter-tuning improves log-marginal likelihood. """
for kernel in kernels:
if kernel == fixed_kernel: continue
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert_greater(gpc.log_marginal_likelihood(gpc.kernel_.theta),
gpc.log_marginal_likelihood(kernel.theta))
def test_predict_consistent():
""" Check binary predict decision has also predicted probability above 0.5.
"""
for kernel in kernels:
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert_array_equal(gpc.predict(X),
gpc.predict_proba(X)[:, 1] >= 0.5)
def build_classifier_gp(data, labels, **kwargs):
linear_kernel = Sum(k1=Product(k1=DotProduct(sigma_0=0, sigma_0_bounds='fixed'), k2=ConstantKernel()),
k2=ConstantKernel())
gp_clf = GaussianProcessClassifier(kernel=linear_kernel)
gp_clf.fit(data, labels)
id_pos_class = gp_clf.classes_ == labels.max()
return gp_clf, gp_clf.predict_proba(data)[:, id_pos_class]
def test_converged_to_local_maximum(kernel):
# Test that we are in local maximum after hyperparameter-optimization.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
lml, lml_gradient = \
gpc.log_marginal_likelihood(gpc.kernel_.theta, True)
assert np.all((np.abs(lml_gradient) < 1e-4) |
(gpc.kernel_.theta == gpc.kernel_.bounds[:, 0]) |
(gpc.kernel_.theta == gpc.kernel_.bounds[:, 1]))
def test_multi_class(kernel):
# Test GPC for multi-class classification problems.
gpc = GaussianProcessClassifier(kernel=kernel)
gpc.fit(X, y_mc)
y_prob = gpc.predict_proba(X2)
assert_almost_equal(y_prob.sum(1), 1)
y_pred = gpc.predict(X2)
assert_array_equal(np.argmax(y_prob, 1), y_pred)
def test_lml_gradient(kernel):
# Compare analytic and numeric gradient of log marginal likelihood.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
lml, lml_gradient = gpc.log_marginal_likelihood(kernel.theta, True)
lml_gradient_approx = \
approx_fprime(kernel.theta,
lambda theta: gpc.log_marginal_likelihood(theta,
False),
1e-10)
assert_almost_equal(lml_gradient, lml_gradient_approx, 3)
def test_multi_class_n_jobs(kernel):
# Test that multi-class GPC produces identical results with n_jobs>1.
gpc = GaussianProcessClassifier(kernel=kernel)
gpc.fit(X, y_mc)
gpc_2 = GaussianProcessClassifier(kernel=kernel, n_jobs=2)
gpc_2.fit(X, y_mc)
y_prob = gpc.predict_proba(X2)
y_prob_2 = gpc_2.predict_proba(X2)
assert_almost_equal(y_prob, y_prob_2)
def test_random_starts():
# Test that an increasing number of random-starts of GP fitting only
# increases the log marginal likelihood of the chosen theta.
n_samples, n_features = 25, 2
rng = np.random.RandomState(0)
X = rng.randn(n_samples, n_features) * 2 - 1
y = (np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1)) > 0
kernel = C(1.0, (1e-2, 1e2)) \
* RBF(length_scale=[1e-3] * n_features,
length_scale_bounds=[(1e-4, 1e+2)] * n_features)
last_lml = -np.inf
for n_restarts_optimizer in range(5):
gp = GaussianProcessClassifier(
kernel=kernel, n_restarts_optimizer=n_restarts_optimizer,
random_state=0).fit(X, y)
lml = gp.log_marginal_likelihood(gp.kernel_.theta)
assert_greater(lml, last_lml - np.finfo(np.float32).eps)
last_lml = lml
def test_custom_optimizer(kernel):
# Test that GPC can use externally defined optimizers.
# Define a dummy optimizer that simply tests 50 random hyperparameters
def optimizer(obj_func, initial_theta, bounds):
rng = np.random.RandomState(0)
theta_opt, func_min = \
initial_theta, obj_func(initial_theta, eval_gradient=False)
for _ in range(50):
theta = np.atleast_1d(rng.uniform(np.maximum(-2, bounds[:, 0]),
np.minimum(1, bounds[:, 1])))
f = obj_func(theta, eval_gradient=False)
if f < func_min:
theta_opt, func_min = theta, f
return theta_opt, func_min
gpc = GaussianProcessClassifier(kernel=kernel, optimizer=optimizer)
gpc.fit(X, y_mc)
# Checks that optimizer improved marginal likelihood
assert_greater(gpc.log_marginal_likelihood(gpc.kernel_.theta),
gpc.log_marginal_likelihood(kernel.theta))
def trainModel(subjectid):
# Load training data from the file matlab generates
traindata = np.genfromtxt('csvdata/' + subjectid +
'_sim.csv', delimiter=',',
missing_values=['NaN', 'nan'],
filling_values=None)
trainx, trainy = cleandata(traindata, downsamplefactor=20)
# Train a Gaussian Process
anisokern = kernels.RBF() # default kernel
gp = GaussianProcessClassifier(kernel=anisokern) # Initialize the GPC
gp.fit(trainx, trainy) # train this class on the data
trainx = trainy = None # Discard all training data to preserve memory
# Load test data
testdata = np.genfromtxt('csvdata/' + subjectid +
'_rival.csv', delimiter=',',
missing_values=['NaN', 'nan'],
filling_values=None)
testx, testy = cleandata(testdata, downsamplefactor=4) # clean data
return gp, testx, testy
def return_model(mode, **kwargs):
if inspect.isclass(mode):
assert getattr(
mode, 'fit',
None) is not None, 'Custom model family should have a fit() method'
model = mode(**kwargs)
elif mode == 'logistic':
solver = kwargs.get('solver', 'liblinear')
n_jobs = kwargs.get('n_jobs', None)
C = kwargs.get('C', 1.)
max_iter = kwargs.get('max_iter', 5000)
model = LogisticRegression(solver=solver,
n_jobs=n_jobs,
C=C,
max_iter=max_iter,
random_state=666)
elif mode == 'Tree':
model = DecisionTreeClassifier(random_state=666)
elif mode == 'RandomForest':
n_estimators = kwargs.get('n_estimators', 50)
model = RandomForestClassifier(n_estimators=n_estimators,
random_state=666)
elif mode == 'GB':
n_estimators = kwargs.get('n_estimators', 50)
model = GradientBoostingClassifier(n_estimators=n_estimators,
random_state=666)
elif mode == 'AdaBoost':
n_estimators = kwargs.get('n_estimators', 50)
model = AdaBoostClassifier(n_estimators=n_estimators, random_state=666)
elif mode == 'SVC':
kernel = kwargs.get('kernel', 'rbf')
model = SVC(kernel=kernel, random_state=666)
elif mode == 'LinearSVC':
model = LinearSVC(loss='hinge', random_state=666)
elif mode == 'GP':
model = GaussianProcessClassifier(random_state=666)
elif mode == 'KNN':
n_neighbors = kwargs.get('n_neighbors', 5)
model = KNeighborsClassifier(n_neighbors=n_neighbors)
elif mode == 'NB':
model = MultinomialNB()
elif mode == 'linear':
model = LinearRegression(random_state=666)
elif mode == 'ridge':
alpha = kwargs.get('alpha', 1.0)
model = Ridge(alpha=alpha, random_state=666)
elif 'conv' in mode:
tf.reset_default_graph()
address = kwargs.get('address', 'weights/conv')
hidden_units = kwargs.get('hidden_layer_sizes', [20])
activation = kwargs.get('activation', 'relu')
weight_decay = kwargs.get('weight_decay', 1e-4)
learning_rate = kwargs.get('learning_rate', 0.001)
max_iter = kwargs.get('max_iter', 1000)
dropout = kwargs.get('dropout', 0.)
early_stopping = kwargs.get('early_stopping', 10)
warm_start = kwargs.get('warm_start', False)
batch_size = kwargs.get('batch_size', 256)
kernel_sizes = kwargs.get('kernel_sizes', [5])
strides = kwargs.get('strides', [5])
channels = kwargs.get('channels', [1])
validation_fraction = kwargs.get('validation_fraction', 0.)
global_averaging = kwargs.get('global_averaging', 0.)
optimizer = kwargs.get('optimizer', 'sgd')
if mode == 'conv':
model = CShapNN(mode='classification',
batch_size=batch_size,
max_epochs=max_iter,
learning_rate=learning_rate,
dropout=dropout,
weight_decay=weight_decay,
validation_fraction=validation_fraction,
early_stopping=early_stopping,
optimizer=optimizer,
warm_start=warm_start,
address=address,
hidden_units=hidden_units,
strides=strides,
global_averaging=global_averaging,
kernel_sizes=kernel_sizes,
channels=channels,
random_seed=666)
elif mode == 'conv_reg':
model = CShapNN(mode='regression',
batch_size=batch_size,
max_epochs=max_iter,
learning_rate=learning_rate,
dropout=dropout,
weight_decay=weight_decay,
validation_fraction=validation_fraction,
early_stopping=early_stopping,
optimizer=optimizer,
warm_start=warm_start,
address=address,
hidden_units=hidden_units,
strides=strides,
global_averaging=global_averaging,
kernel_sizes=kernel_sizes,
channels=channels,
random_seed=666)
elif 'NN' in mode:
solver = kwargs.get('solver', 'adam')
hidden_layer_sizes = kwargs.get('hidden_layer_sizes', (20, ))
if isinstance(hidden_layer_sizes, list):
hidden_layer_sizes = list(hidden_layer_sizes)
activation = kwargs.get('activation', 'relu')
learning_rate_init = kwargs.get('learning_rate', 0.001)
max_iter = kwargs.get('max_iter', 5000)
early_stopping = kwargs.get('early_stopping', False)
warm_start = kwargs.get('warm_start', False)
batch_size = kwargs.get('batch_size', 'auto')
if mode == 'NN':
model = MLPClassifier(solver=solver,
hidden_layer_sizes=hidden_layer_sizes,
activation=activation,
learning_rate_init=learning_rate_init,
warm_start=warm_start,
max_iter=max_iter,
early_stopping=early_stopping,
batch_size=batch_size)
if mode == 'NN_reg':
model = MLPRegressor(solver=solver,
hidden_layer_sizes=hidden_layer_sizes,
activation=activation,
learning_rate_init=learning_rate_init,
warm_start=warm_start,
max_iter=max_iter,
early_stopping=early_stopping,
batch_size=batch_size)
else:
raise ValueError("Invalid mode!")
return model
classifier = np.all(np.unique(Y.to_numpy()) == [0, 1])
outputs = Y.shape[1]
# separate the data into training and testing
if TIME_SERIES:
test_idx = X.index.values[-int(X.shape[0] / 5):]
else:
np.random.seed(1)
test_idx = np.random.choice(a=X.index.values,
size=int(X.shape[0] / 5),
replace=False)
train_idx = np.array(list(set(X.index.values) - set(test_idx)))
# set up the model
if classifier:
model = MultiOutputClassifier(GaussianProcessClassifier(random_state=42))
else:
model = MultiOutputRegressor(GaussianProcessRegressor(random_state=42))
# train the model
model.fit(X.iloc[train_idx, :], Y.iloc[train_idx, :])
# In[2]: Collect the predictions
# predict training and testing data
train_predict = pd.DataFrame(model.predict(X.iloc[train_idx, :]),
columns=Y.columns)
test_predict = pd.DataFrame(model.predict(X.iloc[test_idx, :]),
columns=Y.columns)
# reshape all of the predictions into a single table
def grid_search_pipeline(X, y):
""" making pipeline and gridsearchcv"""
clf = Pipeline(steps=[
('scaler', StandardScaler())
, ('clf', LogisticRegression())
])
param_grid = [
{ 'clf': [DecisionTreeClassifier(random_state=24, max_leaf_nodes=1000, class_weight='balanced')],
'clf__max_depth': [5, 7, 10, 11, 12, 13, 14, 15, 16, 17]
},
{ 'clf': [ExtraTreeClassifier(max_features=None, random_state=24)],
'clf__max_depth': [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 30, 50, 100]
},
{ 'clf': [ExtraTreesClassifier(max_features=None, n_jobs=-1, random_state=24)],
'clf__n_estimators': [1, 2, 5, 7, 10, 15, 20],
'clf__max_depth': [2, 4, 6, 8, 10, 12, 15, 25, 30]
},
{ 'clf': [RandomForestClassifier(max_features=None, n_jobs=-1, random_state=24)],
'clf__n_estimators': [61, 62, 63, 64, 65, 66, 67, 68, 69, 70],
'clf__max_depth': [2, 4, 6, 8, 10, 12, 15, 25, 30]
},
{ 'clf': [KNeighborsClassifier(weights='distance', n_jobs = -1)],
'clf__n_neighbors': list(range(1,30)),
'clf__leaf_size': list(range(1,30)) # estimator__ prefix to get through MOC
},
{'clf': [RadiusNeighborsClassifier(weights='distance', n_jobs = -1 )],
'clf__radius': [300, 200, 100],
'clf__leaf_size': [100, 50, 15]
},
{ 'clf': [RidgeClassifier(max_iter=110, class_weight='balanced', random_state=24)],
'clf__alpha': [100000, 50000, 10000, 5000]
},
{ 'clf': [LogisticRegression(penalty='elasticnet', random_state=24, max_iter=1000, n_jobs = -1, l1_ratio=.5)],
'clf__solver': ['saga'],
'clf__multi_class': ['auto', 'ovr', 'multinomial']
},
{ 'clf': [LogisticRegressionCV(penalty='elasticnet', n_jobs=-1, random_state=24, l1_ratios=[.5])],
'clf__solver': ['saga'],
'clf__multi_class': ['auto', 'ovr', 'multinomial']
},
{ 'clf': [RidgeClassifierCV(cv = 5, class_weight='balanced')],
'clf__class_weight': [None, 'balanced']
},
{ 'clf': [GaussianNB()]
},
{ 'clf': [NearestCentroid()],
'clf__shrink_threshold': [6,5,4, 4.5, 3, 3.5, 2,1]
},
{ 'clf': [LinearSVC(dual = False, random_state=24, max_iter=1000000000)],
'clf__C': np.linspace(0,10,200).tolist(),
'clf__multi_class': ['ovr', 'crammer_singer']
}, #based on https://scikit-learn.org/stable/modules/svm.html, param C trades off misclassification of training examples against simplicity of the decision surface. A low C makes the decision surface smooth, while a high C aims at classifying all training examples correctly, exponentially spaced to get good values
{ 'clf': [LinearDiscriminantAnalysis()],
'clf__solver': ['lsqr', 'eigen'],
'clf__shrinkage': [None, 'auto']
},
{ 'clf': [QuadraticDiscriminantAnalysis()]
},
{ 'clf': [LabelPropagation(kernel='knn', gamma=0, n_jobs=-1)],
'clf__n_neighbors': np.arange(1, 21).tolist(),
},
{ 'clf': [LabelSpreading(kernel='knn', gamma=0, n_jobs=-1)],
'clf__n_neighbors': np.arange(1, 21).tolist(),
'clf__alpha': [.2, .4, .6, .8]
},
{ 'clf': [NuSVC(max_iter=-1, random_state=24)],
'clf__nu': [1, 10, 20],
'clf__kernel': ['linear', 'poly','rbf', 'sigmoid', 'precomputed'],
'clf__gamma': ['scale', 'auto'],
'clf__shrinking': [True,False],
'clf__class_weight': [None, 'balanced'],
'clf__decision_function_shape': ['ovo','ovr'],
},
{ 'clf': [SVC(probability=True, random_state=24)],
'clf__C': [.2, .4, .6, .8],
'clf__kernel': ['linear', 'poly','rbf', 'sigmoid', 'precomputed'],
'clf__gamma': ['scale', 'auto'],
'clf__shrinking': [True,False],
'clf__class_weight': [None, 'balanced'],
'clf__decision_function_shape': ['ovo','ovr'],
},
{ 'clf': [GaussianProcessClassifier(copy_X_train=False, random_state=24, n_jobs=-1)],
'clf__n_restarts_optimizer': [1, 2]
},
{ 'clf': [SGDClassifier(n_jobs=-1, random_state=24, )],
'clf__penalty': ['l2', 'elasticnet'],
'clf__learning_rate': ['optimal', 'adaptive'],
'clf__eta0': [0.2, 0.4, 0.6, 0.8, 1, 1.2, 1.4],
'clf__class_weight': [None, 'balanced'],
},
{ 'clf': [Perceptron(n_jobs=-1, random_state=24, )],
'clf__penalty': ['l2', 'elasticnet'],
'clf__class_weight': [None, 'balanced'],
},
{ 'clf': [PassiveAggressiveClassifier(n_jobs=-1, random_state=24, average=True)],
'clf__C': [.001, .01, .1, .2, .4, .6, .8, 1, 1.2],
'clf__class_weight': [None, 'balanced']
},
{ 'clf': [GradientBoostingClassifier(random_state=24)],
'clf__learning_rate': [.06, .07, .08,.09],
'clf__n_estimators': [180, 190, 200],
'clf__max_depth': [2, 4, 6, 8],
'clf__init': [None, 'zero']
},
]
scoring = {'balanced_accuracy': 'balanced_accuracy'
, 'f1_micro': 'f1_micro'
,'f1_macro': 'f1_macro'
, 'f1_weighted': 'f1_weighted'
,'precision_micro': 'precision_micro'
, 'precision_macro': 'precision_macro'
,'precision_weighted': 'precision_weighted'
, 'recall_micro': 'recall_micro'
,'recall_macro': 'recall_macro'
, 'recall_weighted': 'recall_weighted'
}
search = GridSearchCV(clf, param_grid, scoring=scoring, refit= 'balanced_accuracy', n_jobs=-1, cv=5, return_train_score=True)
search.fit(X, y)
print(f'Training Machine Learning Classifier for {SYSTEM} Color Categories: successful!')
return search
def plot(df, options):
UNIQ_GROUPS = df.group.unique()
UNIQ_GROUPS.sort()
sns.set_style("white")
grppal = sns.color_palette("Set2", len(UNIQ_GROUPS))
print '# UNIQ GROUPS', UNIQ_GROUPS
cent_stats = df.groupby(
['position', 'group', 'side']).apply(stats_per_group)
cent_stats.reset_index(inplace=True)
import time
from sklearn import preprocessing
from sklearn.gaussian_process import GaussianProcessRegressor, GaussianProcessClassifier
from sklearn.gaussian_process.kernels import Matern, WhiteKernel, ExpSineSquared, ConstantKernel, RBF
ctlDF = cent_stats[ cent_stats['group'] == 0 ]
TNRightDF = cent_stats[ cent_stats['group'] != 0]
TNRightDF = TNRightDF[TNRightDF['side'] == 'right']
dataDf = pd.concat([ctlDF, TNRightDF], ignore_index=True)
print dataDf
yDf = dataDf['group'] == 0
yDf = yDf.astype(int)
y = yDf.values
print y
print y.shape
XDf = dataDf[['position', 'values']]
X = XDf.values
X = preprocessing.scale(X)
print X
print X.shape
# kernel = ConstantKernel() + Matern(length_scale=mean, nu=3 / 2) + \
# WhiteKernel(noise_level=1e-10)
kernel = 1**2 * Matern(length_scale=1, nu=1.5) + \
WhiteKernel(noise_level=0.1)
figure = plt.figure(figsize=(10, 6))
stime = time.time()
gp = GaussianProcessClassifier(kernel)
gp.fit(X, y)
print gp.kernel_
print gp.log_marginal_likelihood()
print("Time for GPR fitting: %.3f" % (time.time() - stime))
# create a mesh to plot in
h = 0.1
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
np.arange(y_min, y_max, h))
plt.figure(figsize=(10, 5))
# Plot the predicted probabilities. For that, we will assign a color to
# each point in the mesh [x_min, m_max]x[y_min, y_max].
Z = gp.predict_proba(np.c_[xx.ravel(), yy.ravel()])
Z = Z[:,1]
print Z
print Z.shape
# Put the result into a color plot
Z = Z.reshape((xx.shape[0], xx.shape[1]))
print Z.shape
plt.imshow(Z, extent=(x_min, x_max, y_min, y_max), origin="lower")
# Plot also the training points
plt.scatter(X[:, 0], X[:, 1], c=np.array(["r", "g"])[y])
plt.xlabel('position')
plt.ylabel('normalized val')
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.xticks(())
plt.yticks(())
plt.title("%s, LML: %.3f" %
("TN vs. Control", gp.log_marginal_likelihood(gp.kernel_.theta)))
plt.tight_layout()
if options.title:
plt.suptitle(options.title)
if options.output:
plt.savefig(options.output, dpi=150)
if options.is_show:
plt.show()
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
h = .02 # step size in the mesh
names = ["Nearest Neighbors", "Linear SVM", "RBF SVM", "Gaussian Process",
"Decision Tree", "Random Forest", "Neural Net", "AdaBoost",
"Naive Bayes", "QDA"]
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="linear", C=0.025),
SVC(gamma=2, C=1),
GaussianProcessClassifier(1.0 * RBF(1.0)),
DecisionTreeClassifier(max_depth=5),
RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
MLPClassifier(alpha=1, max_iter=1000),
AdaBoostClassifier(),
GaussianNB(),
QuadraticDiscriminantAnalysis()]
# X, y = make_classification(n_features=2, n_redundant=0, n_informative=2,
# random_state=1, n_clusters_per_class=1)
rng = np.random.RandomState(2)
# X += 2 * rng.uniform(size=X.shape)
linearly_separable = (X, y)
def __init__(self, **kwargs):
super(GaussianProcess, self).__init__()
super(GaussianProcess, self).SetModel(
GaussianProcessClassifier(random_state=42, **kwargs))
label = shuffle(label, random_state=41)[:5000]
Kfold = StratifiedKFold(n_splits=n_splits)
accuracy_rbf_training = np.zeros(n_splits)
accuracy_rbf_testing = np.zeros(n_splits)
accuracy_matern_traing = np.zeros(n_splits)
accuracy_matern_testing = np.zeros(n_splits)
nlpd_matern_t = np.zeros(n_splits)
nlpd_matern_v = np.zeros(n_splits)
best_kernel = None
best_nlpd = np.inf
for i, (train_index, test_index) in enumerate(Kfold.split(features, label)):
X_train, X_test = features[train_index], features[test_index]
y_train, y_test = label[train_index], label[test_index]
gp_matern_fix = GaussianProcessClassifier(kernel=3.7**2 * Matern(length_scale=9.4, nu=1.5),
optimizer=None)
gp_matern_fix.fit(X_train,y_train)
accuracy_matern_traing[i] = accuracy_score(y_train, gp_matern_fix.predict(X_train))
accuracy_matern_testing[i] = accuracy_score(y_test, gp_matern_fix.predict(X_test))
neg_lpd_matern_t = -np.mean(np.log(gp_matern_fix.predict_proba(X_train)[np.arange(len(X_train)), y_train]))
neg_lpd_matern_v = -np.mean(np.log(gp_matern_fix.predict_proba(X_test)[np.arange(len(X_test)), y_test]))
nlpd_matern_t[i] = neg_lpd_matern_t
nlpd_matern_v[i] = neg_lpd_matern_v
print("Average training accuracy with matern kernel: %.5f" % np.mean(accuracy_matern_traing))
print("Average testing accuracy with matern kernel: %.5f" % np.mean(accuracy_matern_testing))
print("Average negative log predictive density of training set with matern kernel: %.5f"
% np.mean(nlpd_matern_t))
print("Average negative log predictive density of validation set with matern kernel: %.5f"
% np.mean(nlpd_matern_v))
print("Total elapsed time: %.5f" % (time.time()-elapsed))
aa.write("\n-------------------10-fold corss-validation result--------------------\n")
for i in n_restarts_optimizerR:
for j in max_iter_predictR:
# print("\tthe param is :%s" % i)
# print("\tthe param is :%s" % j)
MCC=[]
ACC=[]
SN =[]
SP=[]
precision=[]
NPV=[]
F1=[]
k=1
for train, test in cv.split(X, y):
gpc=GaussianProcessClassifier(kernel = 1.0 * RBF(1.0),n_restarts_optimizer=i,max_iter_predict=j)
y_true,y_pred =y[test], gpc.fit(X.iloc[train], y[train]).predict(X.iloc[test])
# print("\tmatthews_corrcoef: %1.3f" % metrics.matthews_corrcoef(y_true, y_pred))
MCCv=metrics.matthews_corrcoef(y_true, y_pred)
MCC.append(MCCv)
# print("\taccuracy_score: %1.3f\n" % metrics.accuracy_score(y_true, y_pred))
ACCv=metrics.accuracy_score(y_true, y_pred)
ACC.append(ACCv)
SNv,SPv,precisionv,NPVv,F1v = comE(y_true, y_pred)
## y_true, y_pred
SN.append(SNv)
SP.append(SPv)
precision.append(precisionv)
NPV.append(NPVv)
F1.append(F1v)
#CHALLENGE - ...and train them on our data clf = clf.fit(X, Y) # Calling decision tree classifier and fitting clf1 =tree.DecisionTreeClassifier() clfDT =clf1.fit(X, Y) #Calling support vector machine and fitting clf2 = svm.SVC(probability=True) clfSVC =clf2.fit(X, Y) #Calling KNeighbors classifier and fitting clf3 = KNeighborsClassifier(n_neighbors=3) clfKN =clf3.fit(X, Y) #Calling gaussian_process classifier and fitting clf4 = GaussianProcessClassifier() clfGP = clf4.fit(X, Y) ##Calling MLPClassifier and fitting clf5 = MLPClassifier(learning_rate='constant', learning_rate_init=0.001,) clfMLP = clf5.fit(X, Y) test = [[180, 80, 42]] #Storing results predictionDT = clfDT.predict (test) predictionSVC = clfSVC.predict (test) predictionKN = clfKN.predict (test) predictionGP = clfGP.predict (test) predictionMLP = clfMLP.predict (test) #Storing probabilities
# Prediction time!
linear_models = []
linear_models.append(("LogisticRegression", LogisticRegression()))
kernel_models = []
kernel_models.append(("Linear Support Vector Classifier", LinearSVC()))
kernel_models.append(
("Support Vector Classifier", SVC(kernel="rbf", probability=True)))
kernel_models.append(("Nu Support Vector Classifer", NuSVC(probability=True)))
neighbor_models = [("K-nearest neighbours Ball",
KNeighborsClassifier(algorithm='ball_tree'))]
gaussian_models = [("Gaussian Process", GaussianProcessClassifier())]
deTree_models = [("Decision Tree", DecisionTreeClassifier())]
ensemble_models = []
ensemble_models.append(("Random forest", RandomForestClassifier()))
ensemble_models.append(("AdaBoost", AdaBoostClassifier()))
ensemble_models.append(("GradientBoosting", GradientBoostingClassifier()))
mlpNetwork_models = [("MLP NNetwork", MLPClassifier())]
model_families = [("Linear Models", linear_models),
("Kernel Methods", kernel_models),
("Neighbour", neighbor_models),
("Gaussian Methods", gaussian_models),
("Decision Tree", deTree_models),
def gpc(self, dataset_array, label_array, data_teste):
from sklearn.gaussian_process import GaussianProcessClassifier
clf = GaussianProcessClassifier(max_iter_predict=300)
clf.fit(dataset_array, label_array)
return clf.predict(data_teste)
# Design of experiments
X = np.array([[-4.61611719, -6.00099547],
[4.10469096, 5.32782448],
[0.00000000, -0.50000000],
[-6.17289014, -4.6984743],
[1.3109306, -6.93271427],
[-5.03823144, 3.10584743],
[-2.87600388, 6.74310541],
[5.21301203, 4.26386883]])
# Observations
y = np.array(g(X) > 0, dtype=int)
# Instanciate and fit Gaussian Process Model
kernel = C(0.1, (1e-5, np.inf)) * DotProduct(sigma_0=0.1) ** 2
gp = GaussianProcessClassifier(kernel=kernel)
gp.fit(X, y)
print("Learned kernel: %s " % gp.kernel_)
# Evaluate real function and the predicted probability
res = 50
x1, x2 = np.meshgrid(np.linspace(- lim, lim, res),
np.linspace(- lim, lim, res))
xx = np.vstack([x1.reshape(x1.size), x2.reshape(x2.size)]).T
y_true = g(xx)
y_prob = gp.predict_proba(xx)[:, 1]
y_true = y_true.reshape((res, res))
y_prob = y_prob.reshape((res, res))
# Plot the probabilistic classification iso-values
Test_X_B = Img_Bank[i].Raw_Img[:,:,2].ravel()
Temp_X = np.array((Test_X_R,Test_X_G,Test_X_B)).transpose()
Temp_Y = Img_Bank[i].Raw_Mask.ravel()
Test_X = np.concatenate((Test_X,Temp_X))
Test_Y = np.concatenate((Test_Y,Temp_Y))
#%% KNN learning trial.
from sklearn.neighbors import KNeighborsClassifier
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
#classifier = KNeighborsClassifier(n_neighbors=10, n_jobs=-1, verbose )
kernel = 1.0 * RBF(1.0)
classifier = GaussianProcessClassifier(kernel=kernel, random_state=0)
print('KNN - Learning Started')
classifier.fit(Test_X, Test_Y, )
print('KNN - Learning Finished')
print('KNN - Prediction Started')
Test_photo = 10
Test_X_R = Img_Bank[Test_photo].Raw_Img[:,:,0].ravel()
Test_X_G = Img_Bank[Test_photo].Raw_Img[:,:,1].ravel()
Test_X_B = Img_Bank[Test_photo].Raw_Img[:,:,2].ravel()
Test_X = np.array((Test_X_R,Test_X_G,Test_X_B)).transpose()
Y_Pred = classifier.predict(Test_X)
Y_Pred = np.reshape(Y_Pred,(384,512))
plt.matshow(Y_Pred)
print(classification_report(y_true, y_pred))
print()
"""## GaussianProcessClassifier"""
# Commented out IPython magic to ensure Python compatibility.
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2)
tuned_parameters = [{'kernel':[1.0 * RBF(0.8),1.0 * RBF(0.9),1.0 * RBF(1.0),1.0 * RBF(1.1),1.0 * RBF(1.2),1.0 * RBF(1.3)]}
]
print()
clf = GridSearchCV(
GaussianProcessClassifier(), tuned_parameters, scoring='roc_auc', cv=10
)
clf.fit(X_train, y_train)
print("Best parameters set found on development set:")
print()
print(clf.best_params_)
print()
print("Grid scores on development set:")
print()
means = clf.cv_results_['mean_test_score']
stds = clf.cv_results_['std_test_score']
for mean, std, params in zip(means, stds, clf.cv_results_['params']):
print("%0.3f (+/-%0.03f) for %r"
# % (mean, std * 2, params))
print()
'L2 logistic (Multinomial)':
LogisticRegression(C=C,
penalty='l2',
solver='saga',
multi_class='multinomial',
max_iter=10000),
'L2 logistic (OvR)':
LogisticRegression(C=C,
penalty='l2',
solver='saga',
multi_class='ovr',
max_iter=10000),
'Linear SVC':
SVC(kernel='linear', C=C, probability=True, random_state=0),
'GPC':
GaussianProcessClassifier(kernel)
}
n_classifiers = len(classifiers)
plt.figure(figsize=(n_features * (X[:, 0].max() - X[:, 0].min()),
n_classifiers * (X[:, 1].max() - X[:, 1].min())))
plt.subplots_adjust(bottom=-.1, top=1)
xx = np.linspace(floor(X[:, 0].min()), ceil(X[:, 0].max()), 100)
yy = np.linspace(floor(X[:, 1].min()), ceil(X[:, 1].max()), 100).T
xx, yy = np.meshgrid(xx, yy)
Xfull = np.c_[xx.ravel(), yy.ravel()]
for index, (name, classifier) in enumerate(classifiers.items()):
classifier.fit(X, y)
def get_model(model_or_name, threads=-1, classify=False, seed=0):
regression_models = {
'xgboost': (XGBRegressor(max_depth=6,
n_jobs=threads,
random_state=seed), 'XGBRegressor'),
'lightgbm': (LGBMRegressor(n_jobs=threads,
random_state=seed,
verbose=-1), 'LGBMRegressor'),
'randomforest':
(RandomForestRegressor(n_estimators=100,
n_jobs=threads), 'RandomForestRegressor'),
'adaboost': (AdaBoostRegressor(), 'AdaBoostRegressor'),
'linear': (LinearRegression(), 'LinearRegression'),
'elasticnet': (ElasticNetCV(positive=True), 'ElasticNetCV'),
'lasso': (LassoCV(positive=True), 'LassoCV'),
'ridge': (Ridge(), 'Ridge'),
'xgb.1k': (XGBRegressor(max_depth=6,
n_estimators=1000,
n_jobs=threads,
random_state=seed), 'XGBRegressor.1K'),
'xgb.10k': (XGBRegressor(max_depth=6,
n_estimators=10000,
n_jobs=threads,
random_state=seed), 'XGBRegressor.10K'),
'lgbm.1k': (LGBMRegressor(n_estimators=1000,
n_jobs=threads,
random_state=seed,
verbose=-1), 'LGBMRegressor.1K'),
'lgbm.10k': (LGBMRegressor(n_estimators=10000,
n_jobs=threads,
random_state=seed,
verbose=-1), 'LGBMRegressor.10K'),
'rf.1k':
(RandomForestRegressor(n_estimators=1000,
n_jobs=threads), 'RandomForestRegressor.1K'),
'rf.10k': (RandomForestRegressor(n_estimators=10000, n_jobs=threads),
'RandomForestRegressor.10K')
}
classification_models = {
'xgboost': (XGBClassifier(max_depth=6,
n_jobs=threads,
random_state=seed), 'XGBClassifier'),
'lightgbm': (LGBMClassifier(n_jobs=threads,
random_state=seed,
verbose=-1), 'LGBMClassifier'),
'randomforest':
(RandomForestClassifier(n_estimators=100,
n_jobs=threads), 'RandomForestClassifier'),
'adaboost': (AdaBoostClassifier(), 'AdaBoostClassifier'),
'logistic': (LogisticRegression(), 'LogisticRegression'),
'gaussian': (GaussianProcessClassifier(), 'GaussianProcessClassifier'),
'knn': (KNeighborsClassifier(), 'KNeighborsClassifier'),
'bayes': (GaussianNB(), 'GaussianNB'),
'svm': (SVC(), 'SVC'),
'xgb.1k': (XGBClassifier(max_depth=6,
n_estimators=1000,
n_jobs=threads,
random_state=seed), 'XGBClassifier.1K'),
'xgb.10k': (XGBClassifier(max_depth=6,
n_estimators=10000,
n_jobs=threads,
random_state=seed), 'XGBClassifier.10K'),
'lgbm.1k': (LGBMClassifier(n_estimators=1000,
n_jobs=threads,
random_state=seed,
verbose=-1), 'LGBMClassifier.1K'),
'lgbm.10k': (LGBMClassifier(n_estimators=1000,
n_jobs=threads,
random_state=seed,
verbose=-1), 'LGBMClassifier.10K'),
'rf.1k':
(RandomForestClassifier(n_estimators=1000,
n_jobs=threads), 'RandomForestClassifier.1K'),
'rf.10k': (RandomForestClassifier(n_estimators=10000, n_jobs=threads),
'RandomForestClassifier.10K')
}
if isinstance(model_or_name, str):
if classify:
model_and_name = classification_models.get(model_or_name.lower())
else:
model_and_name = regression_models.get(model_or_name.lower())
if not model_and_name:
raise Exception("unrecognized model: '{}'".format(model_or_name))
else:
model, name = model_and_name
else:
model = model_or_name
name = re.search("\w+", str(model)).group(0)
return model, name
mask = NMBA > 0
NMBA = mask * 1
X = pd.concat([NMBA, Age, Berlin, Sex, Weight], axis=1)
collist = list(X.columns)
imp = Imputer(missing_values='NaN', strategy='most_frequent', axis=0)
imp.fit(X)
X = imp.transform(X)
X = pd.DataFrame(X, columns=collist)
X_train, X_test, Y_train, Y_test = \
train_test_split(X,Y,test_size=0.1,random_state=1)
# Kernel
# myKernel = kernels.Sum(kernels.Matern(), kernels.RBF())
# myKernel = kernels.Sum(myKernel,kernels.RationalQuadratic())
# myKernel = kernels.Sum(myKernel,kernels.DotProduct())
myKernel = kernels.RBF()
myKernel = kernels.Sum(myKernel, kernels.DotProduct())
myKernel = kernels.Sum(myKernel, kernels.ConstantKernel())
# myKernel = kernels.Product(myKernel, kernels.DotProduct())
# myKernel = kernels.Sum(myKernel,kernels.ConstantKernel())
model = GaussianProcessClassifier(kernel=myKernel, warm_start=True, n_jobs=2)
model.fit(X_train, Y_train)
y_pred = model.predict(X_test)
predictions = [round(value) for value in y_pred]
accuracy = accuracy_score(Y_test, predictions)
print(round(accuracy, 2))
# filename = 'gp.pkl'
# pickle.dump(model, open(filename, 'wb'))
maxIter = 1000
tolerance = 1e-3
svc = SVC()
svmLinear = SVC(kernel='linear')
ridge = lm.SGDClassifier(loss='squared_loss',
penalty='l2',
alpha=0.5,
max_iter=maxIter)
logisitc = lm.LogisticRegression()
cartTree = tree.DecisionTreeClassifier()
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
gp = GaussianProcessClassifier(kernel=RBF(), multi_class='one_vs_one')
# Bagging
from sklearn.ensemble import BaggingClassifier
cartTree_bagging = BaggingClassifier(cartTree,
max_samples=0.7,
max_features=1.0)
# Neural Network
from sklearn.neural_network import MLPClassifier
nn = MLPClassifier(solver='lbfgs', alpha=1e-6,\
hidden_layer_sizes=[18,12], random_state=1)
modelList = [['ridge', ridge], ['RBF SVM', svc], ['Linear SVM', svmLinear],
['Logistic', logisitc], ['CART Decision Tree', cartTree],
classifiers = {
'mock': MockClassifier(),
'nbayes': GaussianNB(),
'logistic': LogisticRegression(random_state=42),
#'adao': our.AdaBoostClassifier(n_estimators=200),
'adas': their.AdaBoostClassifier(n_estimators=200, random_state=42),
'forest': RandomForestClassifier(n_estimators=200, random_state=42),
'mlp': MLPClassifier(random_state=42),
'svm': SVC(probability=True, random_state=42),
'knn': KNeighborsClassifier(3),
'svc-linear': SVC(kernel="linear",
C=0.025,
probability=True,
random_state=42),
'svc-rbf': SVC(gamma=2, C=1, probability=True, random_state=42),
'gp': GaussianProcessClassifier(1.0 * RBF(1.0), random_state=42),
'tree': DecisionTreeClassifier(max_depth=5, random_state=42),
'qda': QuadraticDiscriminantAnalysis(reg_param=0.1),
'lda': LinearDiscriminantAnalysis()
}
score_types = {
'mock': 'predict_proba',
'nbayes': 'predict_proba',
'logistic': 'predict_proba',
#'adao': 'predict_proba',
'adas': 'predict_proba',
'forest': 'predict_proba',
'mlp': 'predict_proba',
'svm': 'sigmoid',
'knn': 'predict_proba',
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF, DotProduct
from sklearn import metrics
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
train, test = train_test_split(importation.t, test_size=0.1)
spamtrain, spamtest = train_test_split(importation.valspam, test_size=0.1)
kernels = [1.0 * RBF(length_scale=1.0), 1.0 * DotProduct(sigma_0=1.0)**2]
for i, kernel in enumerate(kernels):
clf = GaussianProcessClassifier(kernel=kernel, warm_start=True)
clf.fit(train, spamtrain)
expected = spamtest
predicted = clf.predict(test)
print(clf.score(train, spamtrain))
#print("Classification report for classifier %s:\n%s\n"
# % (clf, metrics.classification_report(expected, predicted)))
#print("Confusion matrix:\n%s" % metrics.confusion_matrix(expected, predicted))
#print("LA SUITE...\n\n\n\n")
#remained_feature_indices=np.where(mask==1)
selected_features_Oulu_data = Oulu_data_train * mask
#selected_features_Oulu_data_test=Oulu_data_test*mask
print(selected_features_Oulu_data.shape)
print(Oulu_labels_train.shape)
#print(selected_features_Oulu_data_test.shape)
#print(Oulu_labels_test.shape)
#the gussian process classification
kernel = 1.0 * RBF(214)
gpc = GaussianProcessClassifier(kernel=kernel,
n_restarts_optimizer=5,
random_state=None,
multi_class="one_vs_rest",
max_iter_predict=100,
n_jobs=-1)
gpc = gpc.fit(selected_features_Oulu_data, Oulu_labels_train)
print('')
print('accuracy on trainingset:',
gpc.score(selected_features_Oulu_data, Oulu_labels_train))
#print('accuracy on testset of the hospital:',gpc.score(selected_features_Oulu_data_test, Oulu_labels_test))
'''
probs = gpc.predict_proba(selected_features_Oulu_data_test)
probs = probs[:, 1]
auc = roc_auc_score(Oulu_labels_test, probs)
print('AUC: %.2f' % auc)
print(f1_score(Oulu_labels_test, gpc.predict(selected_features_Oulu_data_test), average='weighted'))
'''
def feature_training(feature_path,
results_path,
seg_name,
train_split=90,
cls_set="basic",
save_confusion_matrices=True):
"""
Function to run the complete training pipeline
:param feature_path: Path to the directory containing the NPY files with the features
:param results_path: Path to the directory where the results should be saved
:param seg_name: Name of the segmentation
:param train_split: Percentage of data used for training, e.g. 80
:param cls_set: Either a list of names of classifiers to be used, or a the single name of classifier; if not used
all classifiers will be used
:param save_confusion_matrices: True if the confusion matrices should be used
:return: List of accuracies
"""
split = (train_split / 100, (100 - train_split) / 100)
(train_data, test_data), (train_label,
test_label) = load_features(split, feature_path)
names = [
"Nearest Neighbors", "Linear SVM", "RBF SVM", "Gaussian Process",
"Decision Tree", "Random Forest", "Neural Net", "AdaBoost",
"Naive Bayes", "QDA"
]
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="linear", C=0.025),
SVC(gamma=2, C=1),
GaussianProcessClassifier(1.0 * RBF(1.0)),
DecisionTreeClassifier(max_depth=5),
RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
MLPClassifier(alpha=1, max_iter=1000),
AdaBoostClassifier(),
GaussianNB(),
QuadraticDiscriminantAnalysis()
]
if type(cls_set) == list:
if all(x in names for x in cls_set):
classifiers = [
classifiers[names.index(classifier)] for classifier in cls_set
]
names = cls_set
elif cls_set in names:
classifiers = [classifiers[names.index(cls_set)]]
names = [cls_set]
confusion_matrices, accuracies = features_train_test(
classifiers, train_data, train_label, test_data, test_label)
accuracies = dict(zip(names, accuracies))
if save_confusion_matrices:
confusion_matrices = dict(zip(names, confusion_matrices))
with open(
os.path.join(results_path,
"confusion_matrices_seg_{}".format(seg_name)),
"wb") as matrix_file:
pickle.dump(confusion_matrices, matrix_file)
return accuracies
def trainPredict(subjectid, makeplot=False):
print("testing participant " + subjectid)
# Load training data from the file matlab generates
traindata = np.genfromtxt('csvdata/' + subjectid +
'_sim.csv', delimiter=',',
missing_values=['NaN', 'nan'],
filling_values=None)
# Clean + downsample this data
trainx, trainy = cleandata(traindata, downsamplefactor=20)
# Train a Gaussian Process
anisokern = kernels.RBF() # default kernel
gp = GaussianProcessClassifier(kernel=anisokern) # Initialize the GPC
gp.fit(trainx, trainy) # train this class on the data
trainx = trainy = None # Discard all training data to preserve memory
# load test data
testdata = np.genfromtxt('csvdata/' + subjectid +
'_rival.csv', delimiter=',',
missing_values=['NaN', 'nan'],
filling_values=None)
testx, testy = cleandata(testdata, downsamplefactor=4) # clean data
testdata = None # clear from memory
# work out percentage in percept for each data point:
percentages, nextpercept = assign_percentage(testy)
# get a prediction for all points in the test data:
predicty = gp.predict(testx)
proby = gp.predict_proba(testx)
if makeplot:
summaryplot(participant, testx, testy, predicty, proby, gp)
# Summarise prediction by reported percept
meanprediction = {'mean' + percept:
proby[testy == value, 1].mean()
for percept, value in perceptindices.iteritems()}
predictiondev = {'stdev' + percept:
proby[testy == value, 1].std()
for percept, value in perceptindices.iteritems()}
predictionaccuracy = {'acc' + percept:
(predicty[testy == value] ==
testy[testy == value]).mean()
for percept, value in perceptindices.iteritems()}
# Summarise prediction by percentage in percept
predictioncourse = {'timecourse' + percept + str(cutoff):
proby[(testy == value) &
(percentages < cutoff) &
(percentages > cutoff - 0.1), 1].mean()
for percept, value in perceptindices.iteritems()
for cutoff in np.linspace(0.1, 1, 10)}
# Summarise mixed percept time courses by the next percept
nextcourse = {'nextcourse' + percept + str(cutoff):
proby[(testy == 0) &
(percentages < cutoff) &
(percentages > cutoff - 0.1) &
(nextpercept == perceptindices[percept]), 1].mean()
for percept in ['highfreq', 'lowfreq']
for cutoff in np.linspace(0.1, 1, 10)}
afterdominant = {'after' + percept + "_" + after + "_" + str(cutoff):
proby[(testy == perceptindices[percept]) &
(percentages < cutoff) &
(percentages > cutoff - 0.1) &
(nextpercept == perceptindices[after]), 1].mean()
for percept, after in [('highfreq', 'mixed'),
('highfreq', 'lowfreq'),
('lowfreq', 'mixed'),
('lowfreq', 'highfreq')]
for cutoff in np.linspace(0.1, 1, 10)}
# Only return the summarised data
return meanprediction, predictiondev, predictionaccuracy, \
predictioncourse, nextcourse, afterdominant
def do_machinea_leaning_stuff(train_X, train_Y, test_X, test_Y):
returnValue = []
test_predict_Y = []
# de facut ceva cu acest rezultat
#f_classif(X, y);
#Algoritmi de clasificare
rfc = RandomForestClassifier(n_estimators=100, max_depth=2, random_state=0)
rfc.fit(train_X, train_Y)
test_predict_Y = rfc.predict(test_X)
returnValue.append({
'name':
"RandomForestClassifier",
'score':
rfc.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
etc = ExtraTreesClassifier()
etc.fit(train_X, train_Y)
test_predict_Y = etc.predict(test_X)
returnValue.append({
'name':
"ExtraTreesClassifier",
'score':
etc.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
gpc = GaussianProcessClassifier(random_state=0)
gpc.fit(train_X, train_Y)
test_predict_Y = gpc.predict(test_X)
# TODO : poate folosim si asta print(gpc.predict_proba(test_X))
returnValue.append({
'name':
"GaussianProcessClassifier",
'score':
gpc.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
pac = PassiveAggressiveClassifier(max_iter=1000, random_state=0, tol=1e-3)
pac.fit(train_X, train_Y)
test_predict_Y = pac.predict(test_X)
returnValue.append({
'name':
"PassiveAggressiveClassifier",
'score':
pac.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
rc = RidgeClassifier()
rc.fit(train_X, train_Y)
test_predict_Y = rc.predict(test_X)
returnValue.append({
'name':
"RidgeClassifier",
'score':
rc.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
sgdc = SGDClassifier(max_iter=1000, tol=1e-3)
sgdc.fit(train_X, train_Y)
test_predict_Y = sgdc.predict(test_X)
returnValue.append({
'name':
"SGDClassifier",
'score':
sgdc.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
bnb = BernoulliNB()
bnb.fit(train_X, train_Y)
test_predict_Y = bnb.predict(test_X)
returnValue.append({
'name':
"BernoulliNB",
'score':
bnb.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
knnc = KNeighborsClassifier(n_neighbors=3)
knnc.fit(train_X, train_Y)
test_predict_Y = knnc.predict(test_X)
returnValue.append({
'name':
"KNeighborsClassifier",
'score':
knnc.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
mlpc = MLPClassifier()
mlpc.fit(train_X, train_Y)
test_predict_Y = mlpc.predict(test_X)
returnValue.append({
'name':
"MLPClassifier",
'score':
mlpc.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
label_prop_model = LabelPropagation()
rng = np.random.RandomState(42)
random_unlabeled_points = rng.rand(len(train_Y)) < 0.3
labels = np.copy(train_Y)
labels[random_unlabeled_points] = -1
label_prop_model.fit(train_X, labels)
test_predict_Y = label_prop_model.predict(test_X)
returnValue.append({
'name':
"LabelPropagation",
'score':
label_prop_model.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
lsvc = LinearSVC(random_state=0, tol=1e-5)
lsvc.fit(train_X, train_Y)
test_predict_Y = lsvc.predict(test_X)
returnValue.append({
'name':
"LinearSVC",
'score':
label_prop_model.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
svc = SVC(gamma='auto')
svc.fit(train_X, train_Y)
test_predict_Y = svc.predict(test_X)
returnValue.append({
'name':
"SVC",
'score':
svc.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
dtc = DecisionTreeClassifier(random_state=0)
dtc.fit(train_X, train_Y)
test_predict_Y = dtc.predict(test_X)
returnValue.append({
'name':
"DecisionTreeClassifier",
'score':
dtc.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
cccv = CalibratedClassifierCV()
cccv.fit(train_X, train_Y)
test_predict_Y = cccv.predict(test_X)
returnValue.append({
'name':
"CalibratedClassifierCV",
'score':
cccv.score(test_X, test_Y),
'accuracy_naive':
(test_Y != test_predict_Y).sum() * 1.0 / len(test_predict_Y),
'accuracy_score':
accuracy_score(test_Y, test_predict_Y),
'classification_report':
classification_report(test_Y, test_predict_Y)
})
return returnValue
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF, DotProduct
xx, yy = np.meshgrid(np.linspace(-3, 3, 50),
np.linspace(-3, 3, 50))
rng = np.random.RandomState(0)
X = rng.randn(200, 2)
Y = np.logical_xor(X[:, 0] > 0, X[:, 1] > 0)
# fit the model
plt.figure(figsize=(10, 5))
kernels = [1.0 * RBF(length_scale=1.0), 1.0 * DotProduct(sigma_0=1.0)**2]
for i, kernel in enumerate(kernels):
clf = GaussianProcessClassifier(kernel=kernel, warm_start=True).fit(X, Y)
# plot the decision function for each datapoint on the grid
Z = clf.predict_proba(np.vstack((xx.ravel(), yy.ravel())).T)[:, 1]
Z = Z.reshape(xx.shape)
plt.subplot(1, 2, i + 1)
image = plt.imshow(Z, interpolation='nearest',
extent=(xx.min(), xx.max(), yy.min(), yy.max()),
aspect='auto', origin='lower', cmap=plt.cm.PuOr_r)
contours = plt.contour(xx, yy, Z, levels=[0], linewidths=2,
linetypes='--')
plt.scatter(X[:, 0], X[:, 1], s=30, c=Y, cmap=plt.cm.Paired)
plt.xticks(())
plt.yticks(())
plt.axis([-3, 3, -3, 3])
def GPC():
return GaussianProcessClassifier(kernel)
from matplotlib import pyplot as plt
from sklearn.metrics.classification import accuracy_score, log_loss
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
# Generate data
train_size = 50
rng = np.random.RandomState(0)
X = rng.uniform(0, 5, 100)[:, np.newaxis]
y = np.array(X[:, 0] > 2.5, dtype=int)
# Specify Gaussian Processes with fixed and optimized hyperparameters
gp_fix = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0), optimizer=None)
gp_fix.fit(X[:train_size], y[:train_size])
gp_opt = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0))
gp_opt.fit(X[:train_size], y[:train_size])
print("Log Marginal Likelihood (initial): %.3f" % gp_fix.log_marginal_likelihood(gp_fix.kernel_.theta))
print("Log Marginal Likelihood (optimized): %.3f" % gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
print(
"Accuracy: %.3f (initial) %.3f (optimized)"
% (
accuracy_score(y[:train_size], gp_fix.predict(X[:train_size])),
accuracy_score(y[:train_size], gp_opt.predict(X[:train_size])),
)
)
size(W, HEIGHT+dy+40)
else:
def pltshow(mplpyplot):
mplpyplot.show()
# nodebox section end
# Generate data
train_size = 50
rng = np.random.RandomState(0)
X = rng.uniform(0, 5, 100)[:, np.newaxis]
y = np.array(X[:, 0] > 2.5, dtype=int)
# Specify Gaussian Processes with fixed and optimized hyperparameters
gp_fix = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0),
optimizer=None)
gp_fix.fit(X[:train_size], y[:train_size])
gp_opt = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0))
gp_opt.fit(X[:train_size], y[:train_size])
print("Log Marginal Likelihood (initial): %.3f"
% gp_fix.log_marginal_likelihood(gp_fix.kernel_.theta))
print("Log Marginal Likelihood (optimized): %.3f"
% gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
print("Accuracy: %.3f (initial) %.3f (optimized)"
% (accuracy_score(y[:train_size], gp_fix.predict(X[:train_size])),
accuracy_score(y[:train_size], gp_opt.predict(X[:train_size]))))
print("Log-loss: %.3f (initial) %.3f (optimized)"
% (log_loss(y[:train_size], gp_fix.predict_proba(X[:train_size])[:, 1]),
## TODO: Add any additional arguments that you will need to pass into your model
parser.add_argument('--length_scale', type=float, default=1.0)
parser.add_argument('--kernel_scaling', type=float, default=1.0)
# args holds all passed-in arguments
args = parser.parse_args()
# Read in csv training file
training_dir = args.data_dir
train_data = pd.read_csv(os.path.join(training_dir, "train.csv"),
header=None,
names=None)
# Labels are in the first column
train_y = train_data.iloc[:, 0]
train_x = train_data.iloc[:, 1:]
## --- Your code here --- ##
## TODO: Define a model
model = GaussianProcessClassifier(args.kernel_scaling *
RBF(args.length_scale))
## TODO: Train the model
model.fit(train_x, train_y)
## --- End of your code --- ##
# Save the trained model
joblib.dump(model, os.path.join(args.model_dir, "model.joblib"))
Ntrain_vec = [20, 50, 100, 500, 1000, 1500]
for i in Ntrain_vec:
#N = np.int(1000)
Ntrain = np.int(i)
#Ntest = np.int(100)
Xtrain = X[0:Ntrain - 1, :]
ytrain = y[0:Ntrain - 1]
Xtest = X[Ntrain:N, :]
ytest = y[Ntrain:N]
#kernel = 1.0 * RBF([20.0]) # isotropic kernel #Test error rate = 0.89
kernel = DotProduct(20.0) #Test error rate = 0.14
gpc_rbf = GaussianProcessClassifier(kernel=kernel).fit(Xtrain, ytrain)
yp_train = gpc_rbf.predict(Xtrain)
train_error_rate = np.mean(np.not_equal(yp_train, ytrain))
yp_test = gpc_rbf.predict(Xtest)
test_error_rate = []
test_error_rate.append(np.mean(np.not_equal(yp_test, ytest)))
print('Training error rate')
print(train_error_rate)
print('Test error rate')
print(test_error_rate)
"""
=====================================================
Confusion matrix for GP regression
=====================================================
def test_lml_improving(kernel):
# Test that hyperparameter-tuning improves log-marginal likelihood.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert_greater(gpc.log_marginal_likelihood(gpc.kernel_.theta),
gpc.log_marginal_likelihood(kernel.theta))
logging.basicConfig(level=logging.DEBUG)
h = .02 # step size in the mesh
names = [
"Nearest Neighbors", "Linear SVM", "RBF SVM", "Gaussian Process",
"Decision Tree", "Random Forest", "Neural Net", "AdaBoost", "Naive Bayes",
"QDA", "Custom NN"
]
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="linear", C=0.025),
SVC(gamma=2, C=1),
GaussianProcessClassifier(1.0 * RBF(1.0), warm_start=True),
DecisionTreeClassifier(max_depth=5),
RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
MLPClassifier(alpha=1),
AdaBoostClassifier(),
GaussianNB(),
QuadraticDiscriminantAnalysis(),
NeuralNetwork([
NeuronLayer(10, 2, 'leaky_relu', True),
NeuronLayer(1, 10, 'sigmoid', True)
],
learning_rate=0.1,
step_decay_factor=0.99)
]
X, y = make_classification(n_features=2,
#
# Initiate classifiers
#
names = [
"Nearest Neighbors", "Linear SVM", "RBF SVM", "Gaussian Process",
"Decision Tree", "Random Forest", "Neural Net", "AdaBoost", "Naive Bayes",
"QDA"
]
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="linear", C=0.025),
SVC(gamma=2, C=1),
GaussianProcessClassifier(),
DecisionTreeClassifier(max_depth=15),
RandomForestClassifier(max_depth=15, n_estimators=5, max_features=2),
MLPClassifier(),
AdaBoostClassifier(),
GaussianNB()
]
# iterate over classifiers
for name, clf in zip(names, classifiers):
clf.fit(X_train, y_train)
score = clf.score(X_test, y_test)
print('\n- Classifier:' + name + ' is scoring ' + str(score) + '.')
#
# Keeping the algo that outperforms the other without tweeking
def selectNext(self):
proba_in = np.ones(self.poolData[self.indicesUnknown].shape[0])
proba_in *= self.model.predict_proba(
self.poolData[self.indicesUnknown])[:, 1]
predicted = []
predictedFalse = []
for p in proba_in:
if p < 0.5:
predicted.append(
0
) # if magically we know that the predicted label l is true, we can add it to training set and train
predictedFalse.append(
1
) # if magically we know that the predicted label l is wrong, we add 1-l to training set and train
else:
predicted.append(1)
predictedFalse.append(0)
#so now think every case and train the classifier for each item separately to calculate expected error of classifier
eRightList = []
eWrongList = []
for i in range(len(predicted)):
if i in self.indicesUntrained:
X_train = np.append(self.trainData, [self.poolData[i]], axis=0)
y_trainRight = np.append(self.trainLabels, predicted[i])
y_trainWrong = np.append(self.trainLabels, predictedFalse[i])
else:
X_train = copy.deepcopy(self.trainData)
if self.voteAggMethod == 'majorityVoting':
selectedVote = self.aggregator.majorityVoting(
self.queriedVoteHistory, i)
elif self.voteAggMethod == 'DS':
selectedVote = self.aggregator.DS(
self.queriedVoteHistory, self.queriedVoteOwnersHistory,
i)
else:
print(
"no valid technique is selected, default vote aggregation technique -majotityVoting- is used!"
)
selectedVote = self.aggregator.majorityVoting(
self.queriedVoteHistory, i)
y_trainRight = copy.deepcopy(self.trainLabels)
y_trainWrong = copy.deepcopy(self.trainLabels)
y_trainRight[np.where(self.indicesTrained == i)] = selectedVote
y_trainWrong[np.where(self.indicesTrained == i)] = selectedVote
self.model.fit(X_train, y_trainRight)
scores = cross_val_score(self.model,
X_train,
y_trainRight,
cv=5,
scoring='f1')
eRightList.append(scores.mean())
self.model.fit(X_train, y_trainWrong)
scores = cross_val_score(self.model,
X_train,
y_trainWrong,
cv=5,
scoring='f1')
eWrongList.append(scores.mean())
# update the active learner back
self.model.fit(self.trainData, self.trainLabels)
# creating bootstaps
bootstraps = []
for i in range(10):
size = self.trainData.shape[0]
bootstrap_idx = np.random.choice(range(size), size, replace=True)
bst_x = self.trainData[bootstrap_idx, :]
bst_y = self.trainLabels[bootstrap_idx]
bootstraps.append([bst_x, bst_y])
#defining ML classifiers to use in bootstrapped training sets
clfs = [
LogisticRegression(C=1., solver='lbfgs'),
CalibratedClassifierCV(SVC(kernel="linear", C=0.025), cv=3),
CalibratedClassifierCV(SVC(gamma=2, C=1), cv=3),
GaussianProcessClassifier(1.0 * RBF(1.0)),
CalibratedClassifierCV(DecisionTreeClassifier(max_depth=40), cv=3),
CalibratedClassifierCV(RandomForestClassifier(max_depth=5,
n_estimators=10),
cv=3),
CalibratedClassifierCV(LinearSVC(class_weight='balanced', C=0.1),
cv=3),
AdaBoostClassifier(),
GaussianNB(),
CalibratedClassifierCV(RandomForestClassifier(n_estimators=50,
max_depth=40),
cv=3)
]
#apply multiprocessing to learn predicted label of each item in the pool by each ML classifier
manager = multiprocessing.Manager()
return_dict = manager.dict()
jobs = []
for i in range(10):
p = multiprocessing.Process(target=self.bootstrap_computation,
args=(i, clfs[i], bootstraps[i],
return_dict))
jobs.append(p)
p.start()
for proc in jobs:
proc.join()
predictions = []
for key, value in return_dict.items():
predictions.append(value)
# calculate p_u values for each item
p_u = []
for i in range(len(predicted)):
p_i = 0
for j in range(len(predictions)):
if predicted[i] == predictions[j][i]:
p_i += 1
p_u.append(p_i / 10)
# calculate expected error of each item in the pool
expErrorList = []
for i in range(len(p_u)):
expErrorList.append(eWrongList[i] - p_u[i] *
(eWrongList[i] - eRightList[i]))
# get the indexes of min n_instances
expErrorList = np.array(expErrorList)
idx = np.argsort(expErrorList)
query_idx = idx[0:self.batchSize]
selectedIndex = self.indicesUnknown[query_idx]
# return the selected items that have minimum expected error
return selectedIndex
def test_lml_precomputed():
# Test that lml of optimized kernel is stored correctly.
for kernel in kernels:
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert_almost_equal(gpc.log_marginal_likelihood(gpc.kernel_.theta),
gpc.log_marginal_likelihood(), 7)
Xs_train = []
Labels = []
for _ in range(5):
d = resample(X_train, n_samples=502)
Xs_train.append(d.iloc[:, :-1])
Labels.append(d['labels'])
# # Gaussian Process Regression
# A task that was considerably more difficult was the prediction of MMSE, which itself is a very noisy measure [8]. We used Gaussian Process Regression[5] (GPR) to predict this measure using selected imaging features, demographic features as well as genetic features. The choice of GPR was motivated by the ability of such models to fit very noisy data [4].
# In[21]:
from sklearn.gaussian_process import GaussianProcessClassifier
gp = GaussianProcessClassifier(optimizer='Welch',
multi_class='one_vs_rest',
n_restarts_optimizer=5)
# In[22]:
gpt = []
for i in range(5):
gp.fit(Xs_train[i], Labels[i])
gpt.append(gp.predict(X_test))
# In[23]:
gpc = []
for i in range(len(y_test)):
p = []
for j in gpt:
# Design of experiments
X = np.array([[-4.61611719, -6.00099547],
[4.10469096, 5.32782448],
[0.00000000, -0.50000000],
[-6.17289014, -4.6984743],
[1.3109306, -6.93271427],
[-5.03823144, 3.10584743],
[-2.87600388, 6.74310541],
[5.21301203, 4.26386883]])
# Observations
y = np.array(g(X) > 0, dtype=int)
# Instanciate and fit Gaussian Process Model
kernel = C(0.1, (1e-5, np.inf)) * DotProduct(sigma_0=0.1) ** 2
gp = GaussianProcessClassifier(kernel=kernel)
gp.fit(X, y)
print("Learned kernel: %s " % gp.kernel_)
# Evaluate real function and the predicted probability
res = 50
x1, x2 = np.meshgrid(np.linspace(- lim, lim, res),
np.linspace(- lim, lim, res))
xx = np.vstack([x1.reshape(x1.size), x2.reshape(x2.size)]).T
y_true = g(xx)
y_prob = gp.predict_proba(xx)[:, 1]
y_true = y_true.reshape((res, res))
y_prob = y_prob.reshape((res, res))
# Plot the probabilistic classification iso-values
from sklearn.datasets import load_iris from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF from sklearn.linear_model import LogisticRegression X, y = load_iris(return_X_y=True) kernel = 1.0 * RBF(1.0) gpc = GaussianProcessClassifier(kernel=kernel, random_state=0).fit(X, y) print(gpc.score(X, y)) clf = LogisticRegression(random_state=0, solver='lbfgs', multi_class='multinomial').fit(X, y) print(clf.score(X,y))
