def train(self):
if self.RBM_init:
attr_bbrbm = BBRBM(n_visible=self.attr_size,
n_hidden=self.embdim,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
attr_bbrbm.fit(self.graph.attrmat,
n_epoches=100,
batch_size=500,
shuffle=True,
verbose=True)
attr_W, attr_Bv, attr_Bh = attr_bbrbm.get_weights()
init = tf.global_variables_initializer()
nodes_emb = np.zeros([self.struc_size, self.embdim])
min_loss = np.inf
with tf.Session() as sess:
sess.run(init)
if self.RBM_init:
sess.run(self.attr_W[0].assign(attr_W))
sess.run(self.attr_b[0].assign(attr_Bh))
sess.run(self.attr_W[1].assign(tf.transpose(attr_W)))
sess.run(self.attr_b[1].assign(attr_Bv))
for epoch in range(self.epoch_num):
nodes_emb_tmp = None
loss = 0.0
batches = self.graph.mini_batch()
for batch_id, batch in enumerate(batches):
X_batch, adjmat_batch, attrX_batch, attrmat_batch = batch
feed_dict = {
self.X: X_batch,
self.A: attrX_batch,
self.adjmat: adjmat_batch,
self.attrmat: attrmat_batch
}
_, batch_emb, batch_loss = sess.run(
[self.optimizer, self.attr_node_emb, self.loss],
feed_dict=feed_dict)
loss += batch_loss
if nodes_emb_tmp is None:
nodes_emb_tmp = batch_emb
else:
nodes_emb_tmp = np.vstack((nodes_emb_tmp, batch_emb))
if (loss < min_loss):
print("epoch:%3d\tloss:%.2f\tsave the best result." %
(epoch, loss))
for i, node_emb in enumerate(nodes_emb_tmp):
sample_node = self.graph.indices[i]
nodes_emb[sample_node] = node_emb
sio.savemat(self.embpath, {
'embedding': nodes_emb,
'label': self.graph.label
})
min_loss = loss
else:
print("epoch:%3d\tloss:%.2f\t" % (epoch, loss))
def train_new_fun(args):
data = get_data(args.data_path, args.label_path)
filename = 'weights_{}_{}_{}.pkl'.format(args.n_hidden, args.learning_rate, args.epochs)
bbrbm = BBRBM(n_visible=data.shape[1], n_hidden=args.n_hidden, learning_rate=args.learning_rate,
momentum=args.momentum, use_tqdm=True)
bbrbm.fit(data, n_epoches=args.epochs, batch_size=args.batch_size)
W, vb, hb = bbrbm.get_weights()
print(get_likelihood(data, W, vb, hb))
pkl.dump([W, vb, hb], open(filename, 'wb'))
def train_on_fun(args):
data = get_data(args.data_path, args.label_path)
filename = 'trained_on_{}_'.format(args.epochs) + args.param_path
params = pkl.load(open(args.param_path, 'rb'))
W, vb, hb = params[0], params[1], params[2]
bbrbm = BBRBM(n_visible=data.shape[1], n_hidden=W.shape[1], learning_rate=args.learning_rate,
momentum=args.momentum, use_tqdm=True)
bbrbm.set_weights(W, vb, hb)
bbrbm.fit(data, n_epoches=args.epochs, batch_size=args.batch_size)
W, vb, hb = bbrbm.get_weights()
print(get_likelihood(data, W, vb, hb))
pkl.dump([W, vb, hb], open(filename, 'wb'))
def train_rbm(n_vis,
n_hid,
learning_rate=0.01,
momentum=0.95,
n_epoches=30,
batch_size=100):
rbm = BBRBM(n_vis,
n_hid,
learning_rate=learning_rate,
momentum=momentum,
use_tqdm=True)
rbm.fit(mnist.train.images, n_epoches=n_epoches, batch_size=batch_size)
return rbm
def make_BBRBM(Image_type, image_type):
# path and name of the data
data_path = '../data/'
data_name = 'TBSS_' + Image_type + '_Rawimage_249.mat'
X_data = loadmat(data_path + data_name)
X_data = X_data[Image_type + '_image'].astype(np.float32)
X_s_max = X_data.max(axis=0)
X_data /= X_s_max
# separate validation and testing dataset
n_train = 229
n_val = 20
X_train = X_data[:n_train]
X_val = X_data[-n_val:]
# train the model
bbrbm = BBRBM(n_visible=X_data.shape[1],
n_hidden=1000,
learning_rate=0.01,
momentum=0.9,
use_tqdm=True)
errs, errs_val = bbrbm.fit(X_train, X_val, n_epoches=900, batch_size=20)
# plot the results
plt.plot(errs)
plt.show()
plt.plot(errs_val)
plt.show()
# save the model
save_path = '../models/rbm_' + image_type + '/'
save_name = image_type + '_model'
bbrbm.save_weights(filename=save_path, name=save_name)
return bbrbm
def main():
args = parameter.set_args()
for i in os.listdir(args.target):
# 数据加载
path = os.path.join(args.target, i)
#加载数据
train_x, train_y = toolbox.load_data(
os.path.join(path, 'final_train.out'))
test_x, test_y = toolbox.load_data(os.path.join(
path, 'final_test.out'))
valid_x, valid_y = toolbox.load_data(
os.path.join(path, 'final_valid.out'))
data = np.vstack((train_x, test_x, valid_x))
#debug
bbrbm = BBRBM(n_visible=data.shape[1],
n_hidden=64,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
errs = bbrbm.fit(data, n_epoches=1000, batch_size=10)
#保存文件
print('model name:' + args.m_n + '_{}'.format(i))
bbrbm.save_weights(args.m_weight + '_{}'.format(i),
args.m_n + '_{}'.format(i))
np.save(args.log + '_{}', errs.format(i))
def main():
logging.basicConfig(level=logging.INFO)
(x_train, _), (x_test, _) = tf.keras.datasets.mnist.load_data()
x_train = x_train / 255
x_test = x_test / 255
dataset = tf.data.Dataset.from_tensor_slices(x_train.reshape(-1, 28 * 28).astype(np.float32))
dataset = dataset.shuffle(1024, reshuffle_each_iteration=True)
rbm = BBRBM(n_visible=28 * 28, n_hidden=64)
rbm.fit(dataset, epoches=100, batch_size=10)
for i in np.random.choice(np.arange(x_test.shape[0]), 5, replace=False):
x = x_test[i]
x_tensor = tf.convert_to_tensor(x.reshape(1, 28 * 28), dtype=tf.float32)
x_reconstructed_tensor = rbm.reconstruct(x_tensor)
x_reconstructed = x_reconstructed_tensor.numpy().reshape(28, 28)
Image.fromarray((x * 255).astype(np.uint8)).save(f'{i}_original.png')
Image.fromarray((x_reconstructed * 255).astype(np.uint8)).save(f'{i}_reconstructed.png')
def make_BBRBM_kfold(train_data, valid_data, num_visible, num_hidden, num_epoches, k, save_path, layer_num):
##############% Transform the data with Bernoulli restricted Boltzmann machine
bbrbm = BBRBM(n_visible=num_visible, n_hidden = num_hidden, learning_rate = 0.001, momentum=0.9, use_tqdm=True)
foldName = str(k)+'_fold/'+'layer'+str(layer_num)+'/'
createFolder(save_path+foldName)
if os.listdir(save_path+foldName):
bbrbm.load_weights(filename = save_path+foldName, name = 'layer'+str(layer_num)+'_model')
else :
errs,errs_val = bbrbm.fit(train_data, valid_data, n_epoches = num_epoches, batch_size=20)
plt.plot(errs)
plt.show()
plt.savefig(save_path+foldName+'train.png')
plt.plot(errs_val)
plt.savefig(save_path+foldName+'val.png')
plt.show()
bbrbm.save_weights(filename = save_path+foldName, name = 'layer'+str(layer_num)+'_model')
transform_data_train = np.zeros([train_data.shape[0] , num_hidden]).astype(np.float32)
for i in range(0,train_data.shape[0]):
transform_data_train[i,:] = bbrbm.transform(train_data[i,:].reshape(1,-1))
transform_data_val = bbrbm.transform(valid_data)
return transform_data_train, transform_data_val
def main():
logging.basicConfig(level=logging.INFO)
(x_train, _), (x_test, _) = tf.keras.datasets.mnist.load_data()
x_train = x_train / 255
x_test = x_test / 255
dataset = tf.data.Dataset.from_tensor_slices(
x_train.reshape(-1, 28 * 28).astype(np.float32))
dataset = dataset.shuffle(1024, reshuffle_each_iteration=True)
rbm_1 = BBRBM(n_visible=28 * 28, n_hidden=64)
rbm_2 = BBRBM(n_visible=64, n_hidden=32)
rbm_3 = BBRBM(n_visible=32, n_hidden=2)
epoches = 100
batch_size = 10
rbm_1.fit(dataset, epoches=epoches, batch_size=batch_size)
dataset_2 = transform_dataset(rbm_1, dataset)
rbm_2.fit(dataset_2, epoches=epoches, batch_size=batch_size)
dataset_3 = transform_dataset(rbm_2, dataset_2)
rbm_3.fit(dataset_3, epoches=epoches, batch_size=batch_size)
def encode(x):
hidden_1 = rbm_1.compute_hidden(x)
hidden_2 = rbm_2.compute_hidden(hidden_1)
hidden_3 = rbm_3.compute_hidden(hidden_2)
return hidden_3
dataset_test = tf.data.Dataset.from_tensor_slices(
x_test.reshape(-1, 28 * 28).astype(np.float32))
encoded_test = []
for batch in dataset_test.batch(2048):
batch_encoded_tensor = encode(batch)
batch_encoded = batch_encoded_tensor.numpy()
encoded_test.append(batch_encoded)
encoded_test = np.vstack(encoded_test)
plt.scatter(encoded_test[:, 0], encoded_test[:, 1], alpha=0.5)
plt.savefig('example.png')
def make_BBRBM_layer2(train, validation, num_visible, num_hidden, num_epoches,
lr, save_path):
bbrbm = BBRBM(n_visible=num_visible,
n_hidden=num_hidden,
learning_rate=lr,
momentum=0.9,
use_tqdm=True)
if os.listdir(save_path):
bbrbm.load_weights(filename=save_path, name='fusion_layer')
else:
errs, errs_val = bbrbm.fit(train,
validation,
n_epoches=num_epoches,
batch_size=20)
plt.plot(errs)
plt.show()
plt.plot(errs_val)
plt.show()
# bbrbm.save_weights(filename=save_path, name='fusion_layer')
transform_train = bbrbm.transform(train)
transform_valid = bbrbm.transform(validation)
return transform_train, transform_valid
bb_momentum_1 = 0.95
bb_err_function_1 = 'mse'
bbrbm_1 = BBRBM(n_visible=bb_n_visible_1,
n_hidden=bb_n_hid_1,
learning_rate=bb_learning_rate_1,
momentum=bb_momentum_1,
err_function=bb_err_function_1,
use_tqdm=False)
bb_n_epoches_1 = 10
bb_batch_size_1 = 128
errs_1 = bbrbm_1.fit(data_x=bb_input_data_1,
n_epoches=bb_n_epoches_1,
batch_size=bb_batch_size_1,
shuffle=True,
verbose=True)
bb_filename_1 = 'pretrain_models/bbrbm_1.ckpt'
bb_name_1 = 'rbm'
bbrbm_1.save_weights(filename=bb_filename_1, name=bb_name_1)
#begin pretraining the second Bernoulli-Bernoulli RBM
bb_input_data_2 = bbrbm_1.transform(bb_input_data_1)
bb_input_data_2 = np.array(bb_input_data_2)
bb_n_visible_2 = bb_input_data_2.shape[1]
bb_n_hid_2 = 2048
bb_learning_rate_2 = 0.01
bb_momentum_2 = 0.95
def dbn(first_layer_hidden_nodes,
second_layer_hidden_nodes,
learning_rate,
epoches,
third_layer_hidden_nodes=0,
batch_size=128):
# ---------------------- PRE-TRAINING --------------------
# gets datasets: x_{train,test} are np arrays
# y_{train,test} are categorical dstributions
# over a digit
x_train, x_test, y_train, y_test = get_datasets()
# ---------------------- FIRST LAYER --------------------
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(x_train.shape[1],
first_layer_hidden_nodes)
visible_biases = np.zeros(shape=(x_train.shape[1]))
hidden_biases = np.zeros(shape=(first_layer_hidden_nodes))
# initialize Restricted Boltzman Machine
first_bbrbm = BBRBM(
n_visible=x_train.shape[1],
n_hidden=first_layer_hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
first_bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
first_bbrbm.fit(data_x=x_train,
validation_data_x=x_test,
n_epoches=epoches,
batch_size=batch_size)
rbm_first_layer_weight_matrix = first_bbrbm.get_weight_matrix()
rbm_first_layer_hidden_layer_biases = first_bbrbm.get_hidden_biases()
first_output = np.dot(x_train, rbm_first_layer_weight_matrix)
for i in range(first_output.shape[0]):
first_output[i, :] += rbm_first_layer_hidden_layer_biases
for j in range(first_output.shape[1]):
# sigmoid(relu(x))
first_output[i, j] = sigmoid(relu(first_output[i, j]))
first_test_output = np.dot(x_test, rbm_first_layer_weight_matrix)
for i in range(first_test_output.shape[0]):
first_test_output[i, :] += rbm_first_layer_hidden_layer_biases
for j in range(first_test_output.shape[1]):
# sigmoid(relu(x))
first_test_output[i, j] = sigmoid(relu(first_test_output[i, j]))
# ---------------------- SECOND LAYER --------------------
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(first_layer_hidden_nodes,
second_layer_hidden_nodes)
visible_biases = np.zeros(shape=(first_layer_hidden_nodes))
hidden_biases = np.zeros(shape=(second_layer_hidden_nodes))
# initialize Restricted Boltzman Machine
second_bbrbm = BBRBM(
n_visible=first_layer_hidden_nodes,
n_hidden=second_layer_hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
second_bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
second_bbrbm.fit(data_x=first_output,
validation_data_x=first_test_output,
n_epoches=epoches,
batch_size=batch_size)
rbm_second_layer_weight_matrix = second_bbrbm.get_weight_matrix()
rbm_second_layer_hidden_layer_biases = second_bbrbm.get_hidden_biases()
second_output = np.dot(first_output, rbm_second_layer_weight_matrix)
for i in range(second_output.shape[0]):
second_output[i, :] += rbm_second_layer_hidden_layer_biases
for j in range(second_output.shape[1]):
# sigmoid(relu(x))
second_output[i, j] = sigmoid(relu(second_output[i, j]))
second_test_output = np.dot(first_test_output,
rbm_second_layer_weight_matrix)
for i in range(second_test_output.shape[0]):
second_test_output[i, :] += rbm_second_layer_hidden_layer_biases
for j in range(second_test_output.shape[1]):
# sigmoid(relu(x))
second_test_output[i, j] = sigmoid(relu(second_test_output[i, j]))
# ---------------------- THIRD LAYER --------------------
if third_layer_hidden_nodes > 0:
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(second_layer_hidden_nodes,
third_layer_hidden_nodes)
visible_biases = np.zeros(shape=(second_layer_hidden_nodes))
hidden_biases = np.zeros(shape=(third_layer_hidden_nodes))
# initialize Restricted Boltzman Machine
third_bbrbm = BBRBM(
n_visible=second_layer_hidden_nodes,
n_hidden=third_layer_hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
third_bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
third_bbrbm.fit(data_x=second_output,
validation_data_x=second_test_output,
n_epoches=epoches,
batch_size=batch_size)
rbm_third_layer_weight_matrix = third_bbrbm.get_weight_matrix()
rbm_third_layer_hidden_layer_biases = third_bbrbm.get_hidden_biases()
third_output = np.dot(second_output, rbm_third_layer_weight_matrix)
for i in range(third_output.shape[0]):
third_output[i, :] += rbm_third_layer_hidden_layer_biases
for j in range(third_output.shape[1]):
# sigmoid(relu(x))
third_output[i, j] = sigmoid(relu(third_output[i, j]))
third_test_output = np.dot(second_test_output,
rbm_third_layer_weight_matrix)
for i in range(third_test_output.shape[0]):
third_test_output[i, :] += rbm_third_layer_hidden_layer_biases
for j in range(third_test_output.shape[1]):
# sigmoid(relu(x))
third_test_output[i, j] = sigmoid(relu(third_test_output[i,
j]))
# ------------------------ CLASSIFICATION ---------------------
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=third_output.shape[1],
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(
mean=0.0, stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(third_output,
np.asarray(y_train),
batch_size=batch_size,
epochs=epoches,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(third_test_output, np.asarray(y_test)))
score = model.evaluate(third_test_output,
np.asarray(y_test),
verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
return score[1]
else:
# ------------------------ CLASSIFICATION ---------------------
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=second_output.shape[1],
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(
mean=0.0, stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(second_output,
np.asarray(y_train),
batch_size=batch_size,
epochs=epoches,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(second_test_output, np.asarray(y_test)))
score = model.evaluate(second_test_output,
np.asarray(y_test),
verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
return score[1]
for i in range(19):
b = np.pad(mnist_images1[i].reshape(28, 28), 1, pad_with, padder=0)#add contour of zeros around the image
x = np.where(mnist.train.labels[i] == 1)
#show_digit(mnist_images1[2].reshape(28, 28))
x1 = list(x) # change to list to add 20 offset
x1[0] = x1[0] +20
x =tuple(x1) #revert back to tuple
# print("x is", x[0])
# x[0] = x[0]+[20]
b[x,0] = 1 # add the label coressponding to the image on the right most clolumn [0..10] ,+20 means that we want to have the labels on pixels [20...29]
mnist_labeled[i] = b.flatten() #convert the image to 1D and store it
#test to see if corretly added labels
show_digit(mnist_labeled[5].reshape(30,30))#new_image.reshape(29,29)
for m in range(19):
show_digit(mnist_labeled[20].reshape(30, 30))
#create the BM
bbrbm = BBRBM(n_visible=900, n_hidden=64, learning_rate=0.01, momentum=0.95, use_tqdm=True)
err = bbrbm.fit(mnist_labeled, n_epoches=100000, batch_size=10)
#save the weights
filename = 'weights_class'
name = 'bbrbm_class'
bbrbm.save_weights(filename,name)
def run(first_hidden, second_hidden, learning_rate, epochs, batch_size):
x_train, x_test, y_train, y_test = get_datasets()
# ------------------FIRST HIDDEN-------------------
weight_matrix = generate_weight_matrix(x_train.shape[1], hidden)
visible_biases = np.zeros(shape=(784))
hidden_biases = np.zeros(shape=(first_hidden))
bbrbm = BBRBM(
n_visible=784,
n_hidden=first_hidden,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
errs = bbrbm.fit(x_train, n_epoches=epochs, batch_size=batch_size)
first_weight_matrix = np.asarray(bbrbm.get_weight_matrix())
first_visible_biases = bbrbm.get_visible_biases()
first_hidden_biases = bbrbm.get_hidden_biases()
first_output = np.dot(x_test, weight_matrix)
for i in range(first_output.shape[0]):
first_output[i, :] += first_hidden_biases
for j in range(first_output.shape[1]):
# sigmoid(relu(x))
first_output[i, j] = sigmoid(np.max(0, first_output[i, j]))
# ------------------SECOND HIDDEN-------------------
bbrbm_second = BBRBM(
n_visible=first_hidden,
n_hidden=784,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
trained_outputs = np.ndarray(shape=(x_train.shape[0], x_train.shape[1]))
for i in range(trained_outputs.shape[0]):
trained_outputs[i, :] = bbrbm.reconstruct(x_train[i].reshape(1, -1))
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=784,
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(trained_outputs,
np.asarray(y_train),
batch_size=batch_size,
epochs=epochs,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(x_test, y_test))
score = model.evaluate(x_test, np.asarray(y_test), verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
# vis(x_train, trained_outputs, 20)
trained_outputs = np.dot(x_train, weight_matrix)
for i in range(trained_outputs.shape[0]):
trained_outputs[i, :] += hidden_biases
debug = 0
# test = np.dot(x_test, weight_matrix)
#
# preds = []
# for i in range(test.shape[0]):
#
# test[i,:]+=hidden_biases
# output_weight_matrix = generate_output_weight_matrix(test.shape[1])
# sgd = optimizers.SGD(lr=0.01, decay=0, momentum=0, nesterov=True)
# model = Model(inputs=test, outputs=y_test)
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
debug = 0
train_y = np.array(label_list)[:10]
print("train_data_list.shape", train_x.shape)
print("label_list.shape", train_y.shape)
test_x = train_x[7:]
test_y = train_y[7:]
print("test_data_list.shape", test_x.shape)
print("label_list.shape", test_y.shape)
# RBM
bbrbm = BBRBM(n_visible=train_x.shape[1],
n_hidden=640,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
errs = bbrbm.fit(train_x, n_epoches=2, batch_size=10)
doc_vec = bbrbm.transform(train_x.reshape(train_x.shape[0], -1))
# doc_vec = doc_vec.reshape(train_data_list.shape[0],10,-1)
print("doc_vec.shape", doc_vec.shape)
# tensorflow placeholders
tf_x = tf.placeholder(
tf.float32, [None, TIME_STEP * INPUT_SIZE]) # shape(batch, 64*10)
image = tf.reshape(
tf_x, [-1, TIME_STEP, INPUT_SIZE]) # (batch, height, width, channel)
tf_y = tf.placeholder(tf.int32, [None, 2]) # input y
# RNN
lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_units=32)
#lstm_cell = tf.contrib.rnn.DropoutWrapper(cell=lstm_cell, output_keep_prob=0.75) # Dropout层
outputs, (h_c, h_n) = tf.nn.dynamic_rnn(
import matplotlib.pyplot as plt
import os, sys, inspect
sys.path.insert(1, os.path.join(sys.path[0], '..'))
from tfrbm import BBRBM, GBRBM
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)
mnist_images = mnist.train.images
X_train = np.where(mnist_images > 0, 1, 0) #binrize the pixels
bbrbm = BBRBM(n_visible=784,
n_hidden=64,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
errs = bbrbm.fit(X_train, n_epoches=50, batch_size=10)
plt.plot(errs)
plt.show()
############################
#Test the Reconstruction of the RBM
IMAGE = 1
image = mnist_images[IMAGE]
#crop the imag
#crop dimentions
x = 6
y = 6
a = image.reshape(28, 28)
img = a[0:16, 0:28] #crop the image
#img = cropND(a,(x,y))
#show cropped image
def rbm(hidden_nodes, learning_rate, epoches, batch_size=128):
# ---------------------- PRE-TRAINING --------------------
# gets datasets: x_{train,test} are np arrays
# y_{train,test} are categorical dstributions
# over a digit
x_train, x_test, y_train, y_test = get_datasets()
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(x_train.shape[1], hidden_nodes)
visible_biases = np.zeros(shape=(x_train.shape[1]))
hidden_biases = np.zeros(shape=(hidden_nodes))
# initialize Restricted Boltzman Machine
bbrbm = BBRBM(
n_visible=x_train.shape[1],
n_hidden=hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
bbrbm.fit(data_x=x_train,
validation_data_x=x_test,
n_epoches=epoches,
batch_size=batch_size)
rbm_weight_matrix = bbrbm.get_weight_matrix()
rbm_hidden_layer_biases = bbrbm.get_hidden_biases()
# TODO: Import commands for comparative representation of
# TODO: original input vs reconstruction input
# generate output that will be used as input in the classification stage
output = np.dot(x_train, rbm_weight_matrix)
for i in range(output.shape[0]):
output[i, :] += rbm_hidden_layer_biases
for j in range(output.shape[1]):
# my_sigmoid(relu(x))
output[i, j] = my_sigmoid(relu(output[i, j]))
test_output = np.dot(x_test, rbm_weight_matrix)
for i in range(test_output.shape[0]):
test_output[i, :] += rbm_hidden_layer_biases
for j in range(test_output.shape[1]):
# my_sigmoid(relu(x))
test_output[i, j] = my_sigmoid(relu(test_output[i, j]))
# ------------------------ CLASSIFICATION ---------------------
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=output.shape[1],
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(output,
np.asarray(y_train),
batch_size=batch_size,
epochs=epoches,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(test_output, y_test))
score = model.evaluate(test_output, np.asarray(y_test), verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
return score[1]
# cv2.imshow('image',image)
# cv2.waitKey(0)
Images.append(image)
Images = np.array(Images, np.int64)
print(Images.shape)
print(Images.dtype)
# Images=Images[:,:,:,np.newaxis]
return Images
if __name__ == '__main__':
Images = get_data(
'/home/cooper/PycharmProjects/zuoye/math_theory_work/作业2/images')
mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)
mnist_images = mnist.train.images
bbrbm = BBRBM(n_visible=914 * 990 / 4,
n_hidden=500,
learning_rate=0.001,
momentum=0.95,
use_tqdm=True)
errs = bbrbm.fit(Images, n_epoches=30, batch_size=10)
plt.plot(errs)
# plt.show()
# model=Get_model()
# model.compile(optimizer='sgd',loss=sparse_categorical_crossentropy,metrics=['accuracy'])
# model.fit(Images,Images,batch_size=1,epochs=10,validation_split=0.2,callbacks=[TensorBoard('/home/cooper/PycharmProjects/zuoye/math_theory_work/logs')])
#
import numpy as np
#import pylab as pl
import matplotlib.pyplot as plt
from tfrbm import BBRBM, GBRBM
#from tensorflow.examples.tutorials.mnist import input_data
import input_data
mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)
mnist_images = mnist.train.images
bbrbm = BBRBM(n_visible=784,
n_hidden=500,
learning_rate=1.0,
momentum=0.0,
use_tqdm=True)
#bbrbm = BBRBM(n_visible=64, n_hidden=32, learning_rate=0.01, momentum=0.95, use_tqdm=True)
errs = bbrbm.fit(mnist_images, n_epoches=1, batch_size=100)
plt.plot(errs)
plt.show()
# %%
# read mnist data
mnist = input_data.read_data_sets('../../data/MNIST', one_hot=True)
mnist_train_images = mnist.train.images
mnist_test_images = mnist.test.images
mnist_test_labels = mnist.test.labels
# %%
# fit Bernoulli-Bernoulli RBM model
bbrbm = BBRBM(n_visible=784,
n_hidden=64,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
bbrbm_errs = bbrbm.fit(mnist_train_images, n_epoches=30, batch_size=10)
# %%
# sava trained RBM model
bbrbm.save_weights('../../output/cn/2018-01-17-ising-hopfield-and-rbm/mnist-bbrbm', 'default')
# bbrbm.load_weights('../../output/cn/2018-01-17-ising-hopfield-and-rbm/mnist-bbrbm', 'default')
# %%
# plot bbrbm errors
plt.style.use('ggplot')
plt.plot(bbrbm_errs)
plt.savefig('../../static/cn/2018-01-17-ising-hopfield-and-rbm/bbrbm-mnist-errs.png', bbox_inches='tight')
# %%
# sample test data and reconstruct data
mnist_test_images_samples = np.zeros([10 * 10, 784])
for x in range(num_sim):
for std in [0.1, 0.2]:
print('###' + str(x) + '###')
print('Xavier weight init')
bbrbm_xavier = BBRBM(n_visible=3,
n_hidden=2,
learning_rate=0.05,
momentum=0,
use_tqdm=False,
init_weight_scheme='xavier',
rbmName='Xavier weight init',
stddev_par=std)
# print(bbrbm_xavier.get_weights())
errs = bbrbm_xavier.fit(data, n_epoches=400, batch_size=1)
# plt.plot(errs)
# plt.show()
# print(bbrbm_xavier.get_weights())
bbrbm_xavier.export_to_csv()
# print('Uniform one weight init')
# bbrbm_ones = BBRBM(n_visible=3, n_hidden=2, learning_rate=0.05, momentum=0, use_tqdm=False, init_weight_scheme ='ones', rbmName = 'Uniform one weight init')
# # print(bbrbm_ones.get_weights())
# errs = bbrbm_ones.fit(data, n_epoches=400, batch_size=1)
# # plt.plot(errs)
# # plt.show()
# # print(bbrbm_ones.get_weights())
# bbrbm_ones.export_to_csv()
#from the Hinton guide "If the statistics used for learning are stochastic, the initial