def train_on_fun(args):
data = get_data(args.data_path, args.label_path)
filename = 'trained_on_{}_'.format(args.epochs) + args.param_path
params = pkl.load(open(args.param_path, 'rb'))
W, vb, hb = params[0], params[1], params[2]
bbrbm = BBRBM(n_visible=data.shape[1], n_hidden=W.shape[1], learning_rate=args.learning_rate,
momentum=args.momentum, use_tqdm=True)
bbrbm.set_weights(W, vb, hb)
bbrbm.fit(data, n_epoches=args.epochs, batch_size=args.batch_size)
W, vb, hb = bbrbm.get_weights()
print(get_likelihood(data, W, vb, hb))
pkl.dump([W, vb, hb], open(filename, 'wb'))
def dbn(first_layer_hidden_nodes,
second_layer_hidden_nodes,
learning_rate,
epoches,
third_layer_hidden_nodes=0,
batch_size=128):
# ---------------------- PRE-TRAINING --------------------
# gets datasets: x_{train,test} are np arrays
# y_{train,test} are categorical dstributions
# over a digit
x_train, x_test, y_train, y_test = get_datasets()
# ---------------------- FIRST LAYER --------------------
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(x_train.shape[1],
first_layer_hidden_nodes)
visible_biases = np.zeros(shape=(x_train.shape[1]))
hidden_biases = np.zeros(shape=(first_layer_hidden_nodes))
# initialize Restricted Boltzman Machine
first_bbrbm = BBRBM(
n_visible=x_train.shape[1],
n_hidden=first_layer_hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
first_bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
first_bbrbm.fit(data_x=x_train,
validation_data_x=x_test,
n_epoches=epoches,
batch_size=batch_size)
rbm_first_layer_weight_matrix = first_bbrbm.get_weight_matrix()
rbm_first_layer_hidden_layer_biases = first_bbrbm.get_hidden_biases()
first_output = np.dot(x_train, rbm_first_layer_weight_matrix)
for i in range(first_output.shape[0]):
first_output[i, :] += rbm_first_layer_hidden_layer_biases
for j in range(first_output.shape[1]):
# sigmoid(relu(x))
first_output[i, j] = sigmoid(relu(first_output[i, j]))
first_test_output = np.dot(x_test, rbm_first_layer_weight_matrix)
for i in range(first_test_output.shape[0]):
first_test_output[i, :] += rbm_first_layer_hidden_layer_biases
for j in range(first_test_output.shape[1]):
# sigmoid(relu(x))
first_test_output[i, j] = sigmoid(relu(first_test_output[i, j]))
# ---------------------- SECOND LAYER --------------------
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(first_layer_hidden_nodes,
second_layer_hidden_nodes)
visible_biases = np.zeros(shape=(first_layer_hidden_nodes))
hidden_biases = np.zeros(shape=(second_layer_hidden_nodes))
# initialize Restricted Boltzman Machine
second_bbrbm = BBRBM(
n_visible=first_layer_hidden_nodes,
n_hidden=second_layer_hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
second_bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
second_bbrbm.fit(data_x=first_output,
validation_data_x=first_test_output,
n_epoches=epoches,
batch_size=batch_size)
rbm_second_layer_weight_matrix = second_bbrbm.get_weight_matrix()
rbm_second_layer_hidden_layer_biases = second_bbrbm.get_hidden_biases()
second_output = np.dot(first_output, rbm_second_layer_weight_matrix)
for i in range(second_output.shape[0]):
second_output[i, :] += rbm_second_layer_hidden_layer_biases
for j in range(second_output.shape[1]):
# sigmoid(relu(x))
second_output[i, j] = sigmoid(relu(second_output[i, j]))
second_test_output = np.dot(first_test_output,
rbm_second_layer_weight_matrix)
for i in range(second_test_output.shape[0]):
second_test_output[i, :] += rbm_second_layer_hidden_layer_biases
for j in range(second_test_output.shape[1]):
# sigmoid(relu(x))
second_test_output[i, j] = sigmoid(relu(second_test_output[i, j]))
# ---------------------- THIRD LAYER --------------------
if third_layer_hidden_nodes > 0:
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(second_layer_hidden_nodes,
third_layer_hidden_nodes)
visible_biases = np.zeros(shape=(second_layer_hidden_nodes))
hidden_biases = np.zeros(shape=(third_layer_hidden_nodes))
# initialize Restricted Boltzman Machine
third_bbrbm = BBRBM(
n_visible=second_layer_hidden_nodes,
n_hidden=third_layer_hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
third_bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
third_bbrbm.fit(data_x=second_output,
validation_data_x=second_test_output,
n_epoches=epoches,
batch_size=batch_size)
rbm_third_layer_weight_matrix = third_bbrbm.get_weight_matrix()
rbm_third_layer_hidden_layer_biases = third_bbrbm.get_hidden_biases()
third_output = np.dot(second_output, rbm_third_layer_weight_matrix)
for i in range(third_output.shape[0]):
third_output[i, :] += rbm_third_layer_hidden_layer_biases
for j in range(third_output.shape[1]):
# sigmoid(relu(x))
third_output[i, j] = sigmoid(relu(third_output[i, j]))
third_test_output = np.dot(second_test_output,
rbm_third_layer_weight_matrix)
for i in range(third_test_output.shape[0]):
third_test_output[i, :] += rbm_third_layer_hidden_layer_biases
for j in range(third_test_output.shape[1]):
# sigmoid(relu(x))
third_test_output[i, j] = sigmoid(relu(third_test_output[i,
j]))
# ------------------------ CLASSIFICATION ---------------------
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=third_output.shape[1],
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(
mean=0.0, stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(third_output,
np.asarray(y_train),
batch_size=batch_size,
epochs=epoches,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(third_test_output, np.asarray(y_test)))
score = model.evaluate(third_test_output,
np.asarray(y_test),
verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
return score[1]
else:
# ------------------------ CLASSIFICATION ---------------------
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=second_output.shape[1],
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(
mean=0.0, stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(second_output,
np.asarray(y_train),
batch_size=batch_size,
epochs=epoches,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(second_test_output, np.asarray(y_test)))
score = model.evaluate(second_test_output,
np.asarray(y_test),
verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
return score[1]
def rbm(hidden_nodes, learning_rate, epoches, batch_size=128):
# ---------------------- PRE-TRAINING --------------------
# gets datasets: x_{train,test} are np arrays
# y_{train,test} are categorical dstributions
# over a digit
x_train, x_test, y_train, y_test = get_datasets()
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(x_train.shape[1], hidden_nodes)
visible_biases = np.zeros(shape=(x_train.shape[1]))
hidden_biases = np.zeros(shape=(hidden_nodes))
# initialize Restricted Boltzman Machine
bbrbm = BBRBM(
n_visible=x_train.shape[1],
n_hidden=hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
bbrbm.fit(data_x=x_train,
validation_data_x=x_test,
n_epoches=epoches,
batch_size=batch_size)
rbm_weight_matrix = bbrbm.get_weight_matrix()
rbm_hidden_layer_biases = bbrbm.get_hidden_biases()
# TODO: Import commands for comparative representation of
# TODO: original input vs reconstruction input
# generate output that will be used as input in the classification stage
output = np.dot(x_train, rbm_weight_matrix)
for i in range(output.shape[0]):
output[i, :] += rbm_hidden_layer_biases
for j in range(output.shape[1]):
# my_sigmoid(relu(x))
output[i, j] = my_sigmoid(relu(output[i, j]))
test_output = np.dot(x_test, rbm_weight_matrix)
for i in range(test_output.shape[0]):
test_output[i, :] += rbm_hidden_layer_biases
for j in range(test_output.shape[1]):
# my_sigmoid(relu(x))
test_output[i, j] = my_sigmoid(relu(test_output[i, j]))
# ------------------------ CLASSIFICATION ---------------------
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=output.shape[1],
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(output,
np.asarray(y_train),
batch_size=batch_size,
epochs=epoches,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(test_output, y_test))
score = model.evaluate(test_output, np.asarray(y_test), verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
return score[1]
def run(first_hidden, second_hidden, learning_rate, epochs, batch_size):
x_train, x_test, y_train, y_test = get_datasets()
# ------------------FIRST HIDDEN-------------------
weight_matrix = generate_weight_matrix(x_train.shape[1], hidden)
visible_biases = np.zeros(shape=(784))
hidden_biases = np.zeros(shape=(first_hidden))
bbrbm = BBRBM(
n_visible=784,
n_hidden=first_hidden,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
errs = bbrbm.fit(x_train, n_epoches=epochs, batch_size=batch_size)
first_weight_matrix = np.asarray(bbrbm.get_weight_matrix())
first_visible_biases = bbrbm.get_visible_biases()
first_hidden_biases = bbrbm.get_hidden_biases()
first_output = np.dot(x_test, weight_matrix)
for i in range(first_output.shape[0]):
first_output[i, :] += first_hidden_biases
for j in range(first_output.shape[1]):
# sigmoid(relu(x))
first_output[i, j] = sigmoid(np.max(0, first_output[i, j]))
# ------------------SECOND HIDDEN-------------------
bbrbm_second = BBRBM(
n_visible=first_hidden,
n_hidden=784,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
trained_outputs = np.ndarray(shape=(x_train.shape[0], x_train.shape[1]))
for i in range(trained_outputs.shape[0]):
trained_outputs[i, :] = bbrbm.reconstruct(x_train[i].reshape(1, -1))
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=784,
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(trained_outputs,
np.asarray(y_train),
batch_size=batch_size,
epochs=epochs,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(x_test, y_test))
score = model.evaluate(x_test, np.asarray(y_test), verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
# vis(x_train, trained_outputs, 20)
trained_outputs = np.dot(x_train, weight_matrix)
for i in range(trained_outputs.shape[0]):
trained_outputs[i, :] += hidden_biases
debug = 0
# test = np.dot(x_test, weight_matrix)
#
# preds = []
# for i in range(test.shape[0]):
#
# test[i,:]+=hidden_biases
# output_weight_matrix = generate_output_weight_matrix(test.shape[1])
# sgd = optimizers.SGD(lr=0.01, decay=0, momentum=0, nesterov=True)
# model = Model(inputs=test, outputs=y_test)
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
debug = 0