def extremLM(data, target, checkHiddenPoints, fileName): # Extreme Learning Machine scalar = 10000 X = data.iloc[:, :-1] X_norm = (X - X.mean()) / (X.max() - X.min()) y = data[target] # y_int = y.apply(lambda x: x * scalar) # Need to be fixed! y_int = y * scalar y_int = y_int.apply(np.int64) # print(y_int) elmList = [] if checkHiddenPoints: inter_num = 100 hiddenNum = [1000, 3000, 5000, 6000, 8000, 10000] # number of hidden points in ELM errorList = [] for hidNum in hiddenNum: total_error = 0 for i in range(inter_num): X_train, X_test, y_train, y_test = train_test_split( X_norm, y_int, test_size=0.2) elm = ELM(hid_num=hidNum).fit(X_train, y_train) y_pred = elm.predict(X_test) sum_mean = 0 # print("This is test value:", y_test.values) # print("This is prediction values:", y_pred) for i in range(len(y_pred)): sum_mean += (y_pred[i] - y_test.values[i])**2 sum_erro = (np.sqrt(sum_mean / len(y_pred))) / scalar # calculate RMSE total_error = total_error + sum_erro print("RMSE:", sum_erro) # Root Mean Squared Error, RMSE print("This is average RMSE for ELM:", total_error / inter_num) errorList.append(total_error / inter_num) # Plot x_pos = list(range(len(hiddenNum))) plt.bar(x_pos, errorList, align='center', alpha=0.5) plt.grid() plt.ylabel('Root Mean Squared Error') plt.xticks(x_pos, hiddenNum) plt.title('Different errors based on the number of hidden points') plt.show() else: for j in range(100): X_train, X_test, y_train, y_test = train_test_split(X_norm, y_int, test_size=0.2) elm = ELM(hid_num=6000).fit(X_train, y_train) y_pred = elm.predict(X_test) sum_mean = 0 # print("This is test value:", y_test.values / scalar) print("This is prediction values:", y_pred / scalar) print("This is iteration number: ", j) for i in range(len(y_pred)): sum_mean += (y_pred[i] - y_test.values[i])**2 sum_erro = (np.sqrt(sum_mean / len(y_pred))) / scalar elmList.append(sum_erro) return elmList
def make_experiments_without_extract(X, y): svc_no_extract_scores = [] knn_no_extract_scores = [] gnb_no_extract_scores = [] dt_no_extract_scores = [] mlp_no_extract_scores = [] elm_no_extract_scores = [] for train_index, test_index in skf.split(X, y): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] svc_clf = SVC(random_state=444) knn_clf = KNeighborsClassifier() gnb_clf = GaussianNB() dt_clf = DecisionTreeClassifier(random_state=444) mlp_clf = MLPClassifier(random_state=444) elm_clf = ELM(X_train.shape[1], 1, 1000) svc_pred = svc_clf.fit(X_train, y_train).predict(X_test) knn_pred = knn_clf.fit(X_train, y_train).predict(X_test) gnb_pred = gnb_clf.fit(X_train, y_train).predict(X_test) dt_pred = dt_clf.fit(X_train, y_train).predict(X_test) mlp_pred = mlp_clf.fit(X_train, y_train).predict(X_test) elm_clf.train(X_train, y_train[:, np.newaxis]) elm_pred = elm_clf.predict(X_test) elm_pred = (elm_pred > 0.5).astype(int) svc_no_extract_scores.append(round(accuracy_score(svc_pred, y_test), 2)) knn_no_extract_scores.append(round(accuracy_score(knn_pred, y_test), 2)) gnb_no_extract_scores.append(round(accuracy_score(gnb_pred, y_test), 2)) dt_no_extract_scores.append(round(accuracy_score(dt_pred, y_test), 2)) mlp_no_extract_scores.append(round(accuracy_score(mlp_pred, y_test), 2)) elm_no_extract_scores.append(round(accuracy_score(elm_pred, y_test), 2)) return [ round(np.average(svc_no_extract_scores), 2), round(np.average(knn_no_extract_scores), 2), round(np.average(gnb_no_extract_scores), 2), round(np.average(dt_no_extract_scores), 2), round(np.average(mlp_no_extract_scores), 2), round(np.average(elm_no_extract_scores), 2) ]
class MLELM(ELM): """ Multi Layer Extreme Learning Machine """ def __init__(self, *, hidden_neurons=None, a=1, random_state=None): super().__init__(hidden_neurons=hidden_neurons, a=a, random_state=random_state) self.betas = [] self.elm = None self.out_num = None def __calc_hidden_layer(self, X): """ Args: X np.array input feature vector """ for beta in self.betas: X = np.dot(beta, X.T).T return X def fit(self, X, y): if self.hidden_neurons is None: self.hidden_neurons = 2 * X.shape[1] self.out_num = max(y) X = self._add_bias(X) for hid_num in self.hidden_neurons[:-1]: _X = self.__calc_hidden_layer(X) W = self._random_state.uniform(-1., 1., (hid_num, _X.shape[1])) H = np.linalg.pinv(self._sigmoid(np.dot(W, _X.T))) beta = np.dot(H.T, _X) self.betas.append(beta) _X = self.__calc_hidden_layer(X) self.elm = ELM(hidden_neurons=self.hidden_neurons[-1]) self.elm.fit(_X, y) return self def predict(self, X): X = self.__calc_hidden_layer(self._add_bias(X)) return self.elm.predict(X)
class MLELM(ELM): """ Multi Layer Extreme Learning Machine """ def __init__(self, hidden_units, a=1): self.hidden_units = hidden_units self.betas = [] self.a = a def __calc_hidden_layer(self, X): """ Args: X np.array input feature vector """ for beta in self.betas: X = np.dot(beta, X.T).T return X def fit(self, X, y): self.out_num = max(y) X = self._add_bias(X) for hid_num in self.hidden_units[:-1]: _X = self.__calc_hidden_layer(X) np.random.seed() W = np.random.uniform(-1., 1., (hid_num, _X.shape[1])) _H = np.linalg.pinv(self._sigmoid(np.dot(W, _X.T))) beta = np.dot(_H.T, _X) self.betas.append(beta) _X = self.__calc_hidden_layer(X) self.elm = ELM(hid_num=self.hidden_units[-1]) self.elm.fit(_X, y) return self def predict(self, X): X = self.__calc_hidden_layer(self._add_bias(X)) return self.elm.predict(X)
def make_experiments_with_lda(X, y): for train_index, test_index in skf.split(X, y): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] lda = LinearDiscriminantAnalysis() X_lda = lda.fit_transform(X_train, y_train) X_test_lda = lda.transform(X_test) svc_clf = SVC(random_state=444) knn_clf = KNeighborsClassifier() gnb_clf = GaussianNB() dt_clf = DecisionTreeClassifier(random_state=444) mlp_clf = MLPClassifier(random_state=444) elm_clf = ELM(X_lda.shape[1], 1, 1000) svc_pred = svc_clf.fit(X_lda, y_train).predict(X_test_lda) knn_pred = knn_clf.fit(X_lda, y_train).predict(X_test_lda) gnb_pred = gnb_clf.fit(X_lda, y_train).predict(X_test_lda) dt_pred = dt_clf.fit(X_lda, y_train).predict(X_test_lda) mlp_pred = mlp_clf.fit(X_lda, y_train).predict(X_test_lda) elm_clf.train(X_lda, y_train[:, np.newaxis]) elm_pred = elm_clf.predict(X_test_lda) elm_pred = (elm_pred > 0.5).astype(int) svc_lda_scores.append(round(accuracy_score(svc_pred, y_test), 2)) knn_lda_scores.append(round(accuracy_score(knn_pred, y_test), 2)) gnb_lda_scores.append(round(accuracy_score(gnb_pred, y_test), 2)) dt_lda_scores.append(round(accuracy_score(dt_pred, y_test), 2)) mlp_lda_scores.append(round(accuracy_score(mlp_pred, y_test), 2)) elm_lda_scores.append(round(accuracy_score(elm_pred, y_test), 2)) return [ round(np.average(svc_lda_scores), 2), round(np.average(knn_lda_scores), 2), round(np.average(gnb_lda_scores), 2), round(np.average(dt_lda_scores), 2), round(np.average(mlp_lda_scores), 2), round(np.average(elm_lda_scores), 2) ]
hit_rates = [] no_of_attributes = dataset.shape[1] - 1 no_of_classes = len(dataset[0, no_of_attributes]) # insert bias no_rows = dataset.shape[0] dataset = np.c_[-1 * np.ones(no_rows), dataset] # perceptron = Perceptron(no_of_classes, no_of_attributes, 5, 'logistic') for j in range(0, 20): print("realization %d" % j) train_X, train_y, test_X, test_y = Classifier.train_test_split(dataset) train_X = np.array(train_X, dtype=float) test_X = np.array(test_X, dtype=float) hidden_units = ELM.model_training(no_of_classes, no_of_attributes, train_X, train_y) elm = ELM(no_of_classes, no_of_attributes, hidden_units) elm.train(train_X, train_y) predictions = elm.predict(test_X) hit_rates.append(elm.evaluate(test_y, predictions)) print(elm.confusion_matrix(test_y, predictions)) # Perceptron.plot_decision_boundaries(train_X, train_y, test_X, test_y, perceptron, hidden_neurons, j) print('hit rates: {}'.format(hit_rates)) print('accuracy: {}'.format(np.mean(hit_rates))) print('std: {}'.format(np.std(hit_rates))) # Perceptron.show_plot_decision_boundaries()
best = [[], 0] for i in range(iters): CVO = KFold(n_splits=n_folds, shuffle=True) acc_values = [] for train_index, test_index in CVO.split(X): X_train, X_test = X[train_index], X[test_index] Y_train, Y_test = Y[train_index], Y[test_index] scaler = StandardScaler() X_train = scaler.fit_transform(X_train) X_test = scaler.transform(X_test) elm = ELM(hidden_units=20, activation="log") #elm = ELM(hidden_units = 20, activation="tan") elm.fit(X_train, Y_train) Y_hat = elm.predict(X_test) Y_hat = np.round(Y_hat) acc_values.append( np.sum( np.where( np.argmax(Y_hat, axis=1) == np.argmax(Y_test, axis=1), 1, 0)) / len(Y_test)) if acc_values[-1] > best[1]: best[0] = confusion_matrix(np.argmax(Y_test, axis=1), np.argmax(Y_hat, axis=1)) best[1] = acc_values[-1] accuracy[i] = np.mean(acc_values) print("Accuracy", np.mean(accuracy)) print("Standard Deviation (accuracy)", np.std(accuracy, axis=0))
mse = np.zeros((iters, 1)) rmse = np.zeros((iters, 1)) for i in range(iters): X_train, X_test, Y_train, Y_test = train_test_split(dataset[:, :1], dataset[:, 1], test_size=0.33) Y_train = Y_train.reshape((Y_train.shape[0], 1)) Y_test = Y_test.reshape((Y_test.shape[0], 1)) scaler = StandardScaler() X_train = scaler.fit_transform(X_train) X_test = scaler.transform(X_test) elm = ELM(hidden_units=8) elm.fit(X_train, Y_train) Y_hat = elm.predict(X_test) mse[i] = ((Y_test - Y_hat)**2).mean(axis=0) rmse[i] = mse[i]**(1. / 2) print("Average MSE", np.mean(mse, axis=0)) print("Average RMSE", np.mean(rmse, axis=0)) print("Standard Deviation (MSE)", np.std(mse, axis=0)) print("Standard Deviation (RMSE)", np.std(rmse, axis=0)) xx = dataset[:, 0:1] xx = scaler.transform(xx) yy = dataset[:, 1] fig, ax = plt.subplots() Z = elm.predict(xx) plt.plot(xx, Z, "-", label="ELM output")
#neural_network.add_neuron(9, "linear") neural_network.add_neuron(100, "sigmoid") output_classes = [] print(len(train)) print(datetime.datetime.now()) for item in train.values: #item[:len(item)-1] neural_network.train(item[:len(item) - 1]) output_classes.append(item[len(item) - 1]) neural_network.update_beta(output_classes) #create output_weights print(datetime.datetime.now()) error_values = [] for item in h.test_dataset.values: predicted = neural_network.predict(item[:len(item) - 1]) print(predicted) actual_value = item[len(item) - 1] print(actual_value) error_values.append((actual_value - predicted)**2) #square the error print("MSE (Mean Squared Error): ", mean(error_values)) if debug: print("checking correctness of shapes:") print(neural_network.input_weights[0].shape) print(len(neural_network.input_weights)) print(neural_network.H.shape) print(len(neural_network.bias))
def main(args): # =============================== # Load dataset # =============================== n_classes = 10 (x_train, t_train), (x_test, t_test) = mnist.load_data() # =============================== # Preprocess # =============================== x_train = x_train.astype(np.float32) / 255. x_train = x_train.reshape(-1, 28**2) x_test = x_test.astype(np.float32) / 255. x_test = x_test.reshape(-1, 28**2) t_train = to_categorical(t_train, n_classes).astype(np.float32) t_test = to_categorical(t_test, n_classes).astype(np.float32) # =============================== # Instantiate ELM # =============================== model = ELM( n_input_nodes=28**2, n_hidden_nodes=args.n_hidden_nodes, n_output_nodes=n_classes, loss=args.loss, activation=args.activation, name='elm', ) # =============================== # Training # =============================== model.fit(x_train, t_train) train_loss, train_acc = model.evaluate(x_train, t_train, metrics=['loss', 'accuracy']) print('train_loss: %f' % train_loss) print('train_acc: %f' % train_acc) # =============================== # Validation # =============================== val_loss, val_acc = model.evaluate(x_test, t_test, metrics=['loss', 'accuracy']) print('val_loss: %f' % val_loss) print('val_acc: %f' % val_acc) # =============================== # Prediction # =============================== x = x_test[:10] t = t_test[:10] y = softmax(model.predict(x)) for i in range(len(y)): print('---------- prediction %d ----------' % (i + 1)) class_pred = np.argmax(y[i]) prob_pred = y[i][class_pred] class_true = np.argmax(t[i]) print('prediction:') print('\tclass: %d, probability: %f' % (class_pred, prob_pred)) print('\tclass (true): %d' % class_true) # =============================== # Save model # =============================== print('saving model...') model.save('model.h5') del model # =============================== # Load model # =============================== print('loading model...') model = load_model('model.h5')