def Training_With_Sklearn_MLP(self, wtest):
clf = MLPClassifier(solver='adam',
activation='relu',
hidden_layer_sizes=(50, 50, 50),
alpha=0.001,
max_iter=300,
batch_size=150)
#clf = MLPRegressor(solver='adam', activation='logistic', hidden_layer_sizes=(24,24,24),alpha=0.01,max_iter=300)
train_input, train_lable = self.get_data()
if wtest:
max = 0
arg = 7
for i in range(7, 31):
clf.hidden_layer_sizes = (i, i)
print("Start training data")
clf.fit(train_input, train_lable)
print("Start predict data")
acc = self.test(clf, 1)
if acc > max:
max = acc
arg = i
print(max, arg)
else:
print("Start training data")
clf.fit(train_input, train_lable)
print("Start predict data")
#self.test(clf, 1)
self.test_auc(clf)
return clf
def generate_mlp(self):
"""
Method for generating a Multilayer perceptron
Neural Network using Sklearn's MLP.
"""
mlp = MLPClassifier(max_iter=1000, verbose=False)
self.num_input_neurons = self.X_train[0].size
self.num_output_neurons = 3 # Buy, sell or hold
self.num_hidden_nodes = round(self.num_input_neurons * (2 / 3) +
self.num_output_neurons)
self.num_hn_perlayer = round(self.num_hidden_nodes / 3)
if self.options['gridsearch']:
# Hyper-parameter optimization
parameter_space = {
'hidden_layer_sizes':
[(self.num_hn_perlayer, self.num_hn_perlayer,
self.num_hn_perlayer), (self.num_hidden_nodes, ),
(self.num_hn_perlayer, self.num_hn_perlayer,
self.num_hn_perlayer, self.num_hn_perlayer)],
'activation': ['tanh', 'relu', 'logistic'],
'solver': ['sgd', 'adam'],
'alpha': [0.0001, 0.05],
'learning_rate': ['constant', 'adaptive'],
}
# with warnings.catch_warnings():
# warnings.filterwarnings("ignore", category=ConvergenceWarning, module="sklearn")
print(
"Performing a gridsearch to find the optimal NN hyper-parameters."
)
self.clf = GridSearchCV(mlp,
parameter_space,
n_jobs=-1,
cv=3,
verbose=False)
self.clf.fit(self.X_train, self.y_train)
# Print results to console
print('Best parameters found:\n', self.clf.best_params_)
# print("Grid scores on development set:")
# means = clf.cv_results_['mean_test_score']
# stds = clf.cv_results_['std_test_score']
# for mean, std, params in zip(means, stds, clf.cv_results_['params']):
# print("%0.3f (+/-%0.03f) for %r" % (mean, std * 2, params))
else:
mlp.hidden_layer_sizes = (self.num_hn_perlayer,
self.num_hn_perlayer,
self.num_hn_perlayer)
self.clf = mlp.fit(self.X_train, self.y_train)
self.test_model()
X_train, X_test, y_train, y_test = train_test_split(X,
Y,
test_size=0.20,
random_state=10)
XtrS, params = ml.rescale(X_train)
Xvas, _ = ml.rescale(X_test, params)
from imblearn.under_sampling import RandomUnderSampler
rus = RandomUnderSampler(random_state=42)
X_res, y_res = rus.fit_resample(XtrS, y_train)
mlp = MLPClassifier(solver='sgd', max_iter=2000)
mlp.hidden_layer_sizes = (100, 100, 100)
mlp.activation = 'logistic'
mlp.learning_rate_init = 0.1
mlp.learning_rate = 'adaptive'
mlp.verbose = True
mlp.fit(X_res, y_res)
print(mlp.score(Xvas, y_test))
Xte = np.genfromtxt(
'C:\\Users\\radad\\OneDrive\\Desktop\\cs178\\CS178-Kaggle-Competition\\X_test.txt',
delimiter=None)
Yte = np.vstack((np.arange(Xte.shape[0]), mlp.predict_proba(Xte)[:, 1])).T
np.savetxt(
'C:\\Users\\radad\\OneDrive\\Desktop\\cs178\\CS178-Kaggle-Competition\\Y_submit.txt',
'max_iter':
[maxIter - 100, maxIter - 50, maxIter, maxIter + 50, maxIter + 100]
}
CV_rfc = GridSearchCV(estimator=clf, param_grid=param_grid, cv=5, n_jobs=-1)
CV_rfc.fit(X_train, y_train)
print(CV_rfc.best_params_)
print("--- %s seconds ---" % (time.time() - start_time))
# In[98]:
# Intializing the parameters to fit the classifier on test dataset
hlSize = CV_rfc.best_params_['hidden_layer_sizes']
maxIter = CV_rfc.best_params_['max_iter']
start_time = time.time()
clf = MLPClassifier(solver='lbfgs')
clf.hidden_layer_sizes = (hlSize[0], )
clf.activation = 'relu'
clf.max_iter = maxIter
clf.fit(X_train, y_train)
print("--- %s seconds ---" % (time.time() - start_time))
# In[99]:
y_pred = clf.predict(X_test)
print(y_pred)
# In[101]:
#Import scikit-learn metrics module for accuracy calculation
from sklearn import metrics
def neural_network(df_images, df_labels, no_of_neurons):
'''This function build the neural network and then delegate the accuracy calculation. It also plots the graphs'''
classifier = MLPClassifier(solver="adam",
verbose=True,
early_stopping=False,
max_iter=1000)
classifier.alpha = 0.05
classifier.hidden_layer_sizes = (no_of_neurons, )
classifier.activation = "relu"
classifier.learning_rate_init = 0.1
# splitting the dataset
train_X, test_X, train_y, test_y = train_test_split(
df_images,
df_labels,
test_size=0.2,
random_state=1,
)
# fit the model
classifier.fit(train_X, train_y)
# for calculating accuracy manually
print('\n------------------------accuracies for ' + str(no_of_neurons) +
' neurons----------------------------\n')
train_accuracy, test_accuracy = calculate_accuracy(classifier, test_X,
test_y, train_X,
train_y)
print(
'\n---------------------Class wise accuracies-------------------------\n'
)
classwise_accuracy(classifier, test_X, test_y, train_X, train_y)
# plotting the loss curve vs iterations
loss_values = classifier.loss_curve_
plt.plot(loss_values)
plt.title('loss vs iterations for : ' +
str(classifier.learning_rate_init) +
' learning rate and no of neurons ' + str(no_of_neurons))
plt.xlabel('iterations')
plt.ylabel('loss')
plt.show()
# save the model to disk
current_date_time = time.strftime("%d/%m/%Y") + '_' + time.strftime(
"%H:%M:%S")
filename = 'model_for_' + str(
no_of_neurons) + '_' + current_date_time + '.sav'
joblib.dump(classifier, filename.replace(
'/', '_')) # saving the classifier using joblib library
print('\nmodel saved in file ' + filename + '\n')
# writing on csv file for reporting
row_to_write = []
row_to_write.append([
filename,
str(no_of_neurons),
str(train_accuracy),
str(test_accuracy)
])
with open('report_regul_mnist.csv', 'a') as writeFile:
writer = csv.writer(writeFile)
writer.writerows(row_to_write)
# printing train set classwise accuracy
# classwise_accuracy(classifier, test_X, test_y, train_X, train_y)
compute_confusion_matrix(classifier, df_labels, test_X, test_y)
def neuralnet(dataset=None,
nn_settings=None,
train=False,
save_settings=True,
print_acc=False):
"""
The following function is a wrapper function that will use a neural
network in order to best estimate the protein count of the input
images. The solver used in this neural network is L-BFGS which is a
quasi-Newton method that is theoretically and experimentally
verified to have a faster convergence and works well with low-
dimensional input. L-BFGS does not work well in other settings
because of its high memory requirement and lack of use of
minibatches. There are other methods that can be tried (Adam, SGD
BatchNorm , Adadelta, Adagrad), but L-BFGS was the best choice for
this application and for the time constraint given in producing
this package.
Parameters
----------
dataset : array[object] : dataset to use neural network on
nn_settings : object : classifier data
train : bool : decide on whether to train or test
save_settings : bool : decide on whether to save classifier data
to 'data' folder
print_acc : bool : passes bool to accuracy() to tell function to
print accuracy
Return
------
count : list : The counts of the input dataset
"""
if dataset is None:
dataset = []
count = None
acc = None
train_x = None
train_y = None
value = None
lrn_dataset = None
lrn_cnt = None
if not train:
save_settings = False
assert (nn_settings is not None
or train), "Neural Network should be training " \
"if there is no settings inputed"
count = []
acc = []
if nn_settings is None:
# using lbfgs (explained in docstring)
classifier = MLPClassifier(solver="lbfgs")
# 1 hidden layer with 100 hidden units
classifier.hidden_layer_sizes = (100, )
# Using a tanh activation
classifier.activation = "tanh"
else:
classifier = nn_settings
train_x = []
train_y = []
value = []
lrn_dataset = []
lrn_cnt = []
for k in range(len(dataset)):
value.append([])
for i in range(len(dataset[k].heights) // 3):
value[k].append(dataset[k].heights[i * 3])
for k in range(len(dataset)):
lrn_dataset.append([])
lrn_cnt.append(dataset[k].count)
for ht_val in value[k]:
lrn_dataset[k].append(ht_val[0])
train_x = lrn_dataset
train_y = lrn_cnt
if train:
classifier.fit(train_x, train_y)
if save_settings:
save_objects([classifier], name='classifier_info.dat')
count = classifier.predict(train_x)
acc.append(accuracy(train_x, count, classifier, output=print_acc))
if train:
print("Training complete")
return
return count
print('test\n', classification_report(
Y_test,
preds)) #logistic regression is able to predict all of the test examples
ho_predictions = logreg.predict(X_val)
output = pd.DataFrame(pass_test)
output['Survived'] = pd.DataFrame(ho_predictions)
output.to_csv('csv_to_submit_lr.csv', index=False)
# In[33]:
from sklearn.neural_network import MLPClassifier
classifier = MLPClassifier(solver="lbfgs")
classifier.hidden_layer_sizes = (
5) # Remember funny notation for tuple with single element
classifier.activation = "relu"
classifier.learning_rate_init = 1
classifier.max_iter = 100
f = classifier.fit(X_train, Y_train)
print(f)
predictions = classifier.predict(X_test)
cm = confusion_matrix(Y_test, predictions)
print(cm)
print('Accuracy = ', ((cm[0, 0] + cm[1, 1]) / np.sum(cm)))
print('train\n', classification_report(Y_train, classifier.predict(X_train)))
print('test\n', classification_report(Y_test, predictions))
ho_predictions = classifier.predict(X_val)
output = pd.DataFrame(pass_test)
output['Survived'] = pd.DataFrame(ho_predictions)
