def main():
args = parameter.set_args()
for i in os.listdir(args.target):
# 数据加载
path = os.path.join(args.target, i)
#加载数据
train_x, train_y = toolbox.load_data(
os.path.join(path, 'final_train.out'))
test_x, test_y = toolbox.load_data(os.path.join(
path, 'final_test.out'))
valid_x, valid_y = toolbox.load_data(
os.path.join(path, 'final_valid.out'))
data = np.vstack((train_x, test_x, valid_x))
#debug
bbrbm = BBRBM(n_visible=data.shape[1],
n_hidden=64,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
errs = bbrbm.fit(data, n_epoches=1000, batch_size=10)
#保存文件
print('model name:' + args.m_n + '_{}'.format(i))
bbrbm.save_weights(args.m_weight + '_{}'.format(i),
args.m_n + '_{}'.format(i))
np.save(args.log + '_{}', errs.format(i))
def make_BBRBM(Image_type, image_type):
# path and name of the data
data_path = '../data/'
data_name = 'TBSS_' + Image_type + '_Rawimage_249.mat'
X_data = loadmat(data_path + data_name)
X_data = X_data[Image_type + '_image'].astype(np.float32)
X_s_max = X_data.max(axis=0)
X_data /= X_s_max
# separate validation and testing dataset
n_train = 229
n_val = 20
X_train = X_data[:n_train]
X_val = X_data[-n_val:]
# train the model
bbrbm = BBRBM(n_visible=X_data.shape[1],
n_hidden=1000,
learning_rate=0.01,
momentum=0.9,
use_tqdm=True)
errs, errs_val = bbrbm.fit(X_train, X_val, n_epoches=900, batch_size=20)
# plot the results
plt.plot(errs)
plt.show()
plt.plot(errs_val)
plt.show()
# save the model
save_path = '../models/rbm_' + image_type + '/'
save_name = image_type + '_model'
bbrbm.save_weights(filename=save_path, name=save_name)
return bbrbm
def train(self):
if self.RBM_init:
attr_bbrbm = BBRBM(n_visible=self.attr_size,
n_hidden=self.embdim,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
attr_bbrbm.fit(self.graph.attrmat,
n_epoches=100,
batch_size=500,
shuffle=True,
verbose=True)
attr_W, attr_Bv, attr_Bh = attr_bbrbm.get_weights()
init = tf.global_variables_initializer()
nodes_emb = np.zeros([self.struc_size, self.embdim])
min_loss = np.inf
with tf.Session() as sess:
sess.run(init)
if self.RBM_init:
sess.run(self.attr_W[0].assign(attr_W))
sess.run(self.attr_b[0].assign(attr_Bh))
sess.run(self.attr_W[1].assign(tf.transpose(attr_W)))
sess.run(self.attr_b[1].assign(attr_Bv))
for epoch in range(self.epoch_num):
nodes_emb_tmp = None
loss = 0.0
batches = self.graph.mini_batch()
for batch_id, batch in enumerate(batches):
X_batch, adjmat_batch, attrX_batch, attrmat_batch = batch
feed_dict = {
self.X: X_batch,
self.A: attrX_batch,
self.adjmat: adjmat_batch,
self.attrmat: attrmat_batch
}
_, batch_emb, batch_loss = sess.run(
[self.optimizer, self.attr_node_emb, self.loss],
feed_dict=feed_dict)
loss += batch_loss
if nodes_emb_tmp is None:
nodes_emb_tmp = batch_emb
else:
nodes_emb_tmp = np.vstack((nodes_emb_tmp, batch_emb))
if (loss < min_loss):
print("epoch:%3d\tloss:%.2f\tsave the best result." %
(epoch, loss))
for i, node_emb in enumerate(nodes_emb_tmp):
sample_node = self.graph.indices[i]
nodes_emb[sample_node] = node_emb
sio.savemat(self.embpath, {
'embedding': nodes_emb,
'label': self.graph.label
})
min_loss = loss
else:
print("epoch:%3d\tloss:%.2f\t" % (epoch, loss))
def train_new_fun(args):
data = get_data(args.data_path, args.label_path)
filename = 'weights_{}_{}_{}.pkl'.format(args.n_hidden, args.learning_rate, args.epochs)
bbrbm = BBRBM(n_visible=data.shape[1], n_hidden=args.n_hidden, learning_rate=args.learning_rate,
momentum=args.momentum, use_tqdm=True)
bbrbm.fit(data, n_epoches=args.epochs, batch_size=args.batch_size)
W, vb, hb = bbrbm.get_weights()
print(get_likelihood(data, W, vb, hb))
pkl.dump([W, vb, hb], open(filename, 'wb'))
def train_rbm(n_vis,
n_hid,
learning_rate=0.01,
momentum=0.95,
n_epoches=30,
batch_size=100):
rbm = BBRBM(n_vis,
n_hid,
learning_rate=learning_rate,
momentum=momentum,
use_tqdm=True)
rbm.fit(mnist.train.images, n_epoches=n_epoches, batch_size=batch_size)
return rbm
def main():
rbm_1 = BBRBM(n_visible=32, n_hidden=64)
x = tf.convert_to_tensor(np.random.random((1, 32)), dtype=tf.float32)
with open('rbm.pickle', 'wb') as f:
pickle.dump(rbm_1, f)
with open('rbm.pickle', 'rb') as f:
rbm_2 = pickle.load(f)
y_1 = rbm_1.compute_hidden(x)
y_2 = rbm_2.compute_hidden(x)
assert np.allclose(y_1.numpy(), y_2.numpy())
def dbn(rbm_hidden_num,
rbm_visible_size,
rbm_hidden_size,
rbm_x,
rbm_type='GBRBM'):
weights = []
biases = []
for i in range(rbm_hidden_num):
# 训练rbm
if i == 0 and rbm_type == 'GBRBM':
rbm = GBRBM(n_visible=rbm_visible_size,
n_hidden=rbm_hidden_size,
learning_rate=0.01,
momentum=0.95,
use_tqdm=False)
else:
rbm = BBRBM(n_visible=rbm_visible_size,
n_hidden=rbm_hidden_size,
learning_rate=0.01,
momentum=0.95,
use_tqdm=False)
errs = rbm.fit(rbm_x, n_epoches=10, batch_size=100, verbose=True)
rbm_x = rbm.transform(rbm_x)
rbm_w, vb, rbm_b = rbm.get_weights()
rbm_visible_size = rbm_hidden_size
weights.append(rbm_w)
biases.append(rbm_b)
return weights, biases
def main():
logging.basicConfig(level=logging.INFO)
(x_train, _), (x_test, _) = tf.keras.datasets.mnist.load_data()
x_train = x_train / 255
x_test = x_test / 255
dataset = tf.data.Dataset.from_tensor_slices(x_train.reshape(-1, 28 * 28).astype(np.float32))
dataset = dataset.shuffle(1024, reshuffle_each_iteration=True)
rbm = BBRBM(n_visible=28 * 28, n_hidden=64)
rbm.fit(dataset, epoches=100, batch_size=10)
for i in np.random.choice(np.arange(x_test.shape[0]), 5, replace=False):
x = x_test[i]
x_tensor = tf.convert_to_tensor(x.reshape(1, 28 * 28), dtype=tf.float32)
x_reconstructed_tensor = rbm.reconstruct(x_tensor)
x_reconstructed = x_reconstructed_tensor.numpy().reshape(28, 28)
Image.fromarray((x * 255).astype(np.uint8)).save(f'{i}_original.png')
Image.fromarray((x_reconstructed * 255).astype(np.uint8)).save(f'{i}_reconstructed.png')
def DBN_DNN(inp, nClasses, depth, width, batch_size=2048):
RBMs = []
weights = []
bias = []
# batch_size = inp.shape
nEpoches = 5
if len(inp.shape) == 3:
inp = inp.reshape((inp.shape[0] * inp.shape[1], inp.shape[2]))
sigma = np.std(inp)
# sigma = 1
rbm = GBRBM(n_visible=inp.shape[-1],
n_hidden=width,
learning_rate=0.002,
momentum=0.90,
use_tqdm=True,
sample_visible=True,
sigma=sigma)
rbm.fit(inp, n_epoches=15, batch_size=batch_size, shuffle=True)
RBMs.append(rbm)
for i in range(depth - 1):
print 'training DBN layer', i
rbm = BBRBM(n_visible=width,
n_hidden=width,
learning_rate=0.02,
momentum=0.90,
use_tqdm=True)
for e in range(nEpoches):
batch_size *= 1 + (e * 0.5)
n_batches = (inp.shape[-2] / batch_size) + (
1 if inp.shape[-2] % batch_size != 0 else 0)
for j in range(n_batches):
stdout.write("\r%d batch no %d/%d epoch no %d/%d" %
(int(time.time()), j + 1, n_batches, e, nEpoches))
stdout.flush()
b = np.array(inp[j * batch_size:min((j + 1) *
batch_size, inp.shape[0])])
for r in RBMs:
b = r.transform(b)
rbm.partial_fit(b)
RBMs.append(rbm)
for r in RBMs:
(W, _, Bh) = r.get_weights()
weights.append(W)
bias.append(Bh)
model = mlp1(x_train.shape[1], nClasses, depth - 1, width)
print len(weights), len(model.layers)
assert len(weights) == len(model.layers) - 1
for i in range(len(weights)):
W = [weights[i], bias[i]]
model.layers[i].set_weights(W)
return model
def train_on_fun(args):
data = get_data(args.data_path, args.label_path)
filename = 'trained_on_{}_'.format(args.epochs) + args.param_path
params = pkl.load(open(args.param_path, 'rb'))
W, vb, hb = params[0], params[1], params[2]
bbrbm = BBRBM(n_visible=data.shape[1], n_hidden=W.shape[1], learning_rate=args.learning_rate,
momentum=args.momentum, use_tqdm=True)
bbrbm.set_weights(W, vb, hb)
bbrbm.fit(data, n_epoches=args.epochs, batch_size=args.batch_size)
W, vb, hb = bbrbm.get_weights()
print(get_likelihood(data, W, vb, hb))
pkl.dump([W, vb, hb], open(filename, 'wb'))
def make_BBRBM_kfold(train_data, valid_data, num_visible, num_hidden, num_epoches, k, save_path, layer_num):
##############% Transform the data with Bernoulli restricted Boltzmann machine
bbrbm = BBRBM(n_visible=num_visible, n_hidden = num_hidden, learning_rate = 0.001, momentum=0.9, use_tqdm=True)
foldName = str(k)+'_fold/'+'layer'+str(layer_num)+'/'
createFolder(save_path+foldName)
if os.listdir(save_path+foldName):
bbrbm.load_weights(filename = save_path+foldName, name = 'layer'+str(layer_num)+'_model')
else :
errs,errs_val = bbrbm.fit(train_data, valid_data, n_epoches = num_epoches, batch_size=20)
plt.plot(errs)
plt.show()
plt.savefig(save_path+foldName+'train.png')
plt.plot(errs_val)
plt.savefig(save_path+foldName+'val.png')
plt.show()
bbrbm.save_weights(filename = save_path+foldName, name = 'layer'+str(layer_num)+'_model')
transform_data_train = np.zeros([train_data.shape[0] , num_hidden]).astype(np.float32)
for i in range(0,train_data.shape[0]):
transform_data_train[i,:] = bbrbm.transform(train_data[i,:].reshape(1,-1))
transform_data_val = bbrbm.transform(valid_data)
return transform_data_train, transform_data_val
def make_BBRBM_layer2(train, validation, num_visible, num_hidden, num_epoches,
lr, save_path):
bbrbm = BBRBM(n_visible=num_visible,
n_hidden=num_hidden,
learning_rate=lr,
momentum=0.9,
use_tqdm=True)
if os.listdir(save_path):
bbrbm.load_weights(filename=save_path, name='fusion_layer')
else:
errs, errs_val = bbrbm.fit(train,
validation,
n_epoches=num_epoches,
batch_size=20)
plt.plot(errs)
plt.show()
plt.plot(errs_val)
plt.show()
# bbrbm.save_weights(filename=save_path, name='fusion_layer')
transform_train = bbrbm.transform(train)
transform_valid = bbrbm.transform(validation)
return transform_train, transform_valid
import numpy as np
#import pylab as pl
import matplotlib.pyplot as plt
from tfrbm import BBRBM, GBRBM
#from tensorflow.examples.tutorials.mnist import input_data
import input_data
mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)
mnist_images = mnist.train.images
bbrbm = BBRBM(n_visible=784,
n_hidden=500,
learning_rate=1.0,
momentum=0.0,
use_tqdm=True)
#bbrbm = BBRBM(n_visible=64, n_hidden=32, learning_rate=0.01, momentum=0.95, use_tqdm=True)
errs = bbrbm.fit(mnist_images, n_epoches=1, batch_size=100)
plt.plot(errs)
plt.show()
plt.locator_params(axis="y", nbins=1)
plt.show()
#labels pixels
accu = [0]
num_avg = 200
n_data = 100 #mnist_images1.shape[0]
print("accuracy", accu)
t1 = np.zeros(10)
print("minist test size", mnist_images1.shape)
#create the BM
bbrbm = BBRBM(n_visible=794,
n_hidden=64,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
#load the saved weights
filename = 'weights_class5kep'
name = 'bbrbm_class5kep'
bbrbm.load_weights(filename, name)
#Test the Reconstruction of the RBM
IMAGE = 20 #26, 31 works well (which is a 6)
#image = mnist_images1[IMAGE]
mask_a_or = np.ones(784)
mask_c_or = np.zeros(784)
#prepare first mask
def dbn(first_layer_hidden_nodes,
second_layer_hidden_nodes,
learning_rate,
epoches,
third_layer_hidden_nodes=0,
batch_size=128):
# ---------------------- PRE-TRAINING --------------------
# gets datasets: x_{train,test} are np arrays
# y_{train,test} are categorical dstributions
# over a digit
x_train, x_test, y_train, y_test = get_datasets()
# ---------------------- FIRST LAYER --------------------
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(x_train.shape[1],
first_layer_hidden_nodes)
visible_biases = np.zeros(shape=(x_train.shape[1]))
hidden_biases = np.zeros(shape=(first_layer_hidden_nodes))
# initialize Restricted Boltzman Machine
first_bbrbm = BBRBM(
n_visible=x_train.shape[1],
n_hidden=first_layer_hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
first_bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
first_bbrbm.fit(data_x=x_train,
validation_data_x=x_test,
n_epoches=epoches,
batch_size=batch_size)
rbm_first_layer_weight_matrix = first_bbrbm.get_weight_matrix()
rbm_first_layer_hidden_layer_biases = first_bbrbm.get_hidden_biases()
first_output = np.dot(x_train, rbm_first_layer_weight_matrix)
for i in range(first_output.shape[0]):
first_output[i, :] += rbm_first_layer_hidden_layer_biases
for j in range(first_output.shape[1]):
# sigmoid(relu(x))
first_output[i, j] = sigmoid(relu(first_output[i, j]))
first_test_output = np.dot(x_test, rbm_first_layer_weight_matrix)
for i in range(first_test_output.shape[0]):
first_test_output[i, :] += rbm_first_layer_hidden_layer_biases
for j in range(first_test_output.shape[1]):
# sigmoid(relu(x))
first_test_output[i, j] = sigmoid(relu(first_test_output[i, j]))
# ---------------------- SECOND LAYER --------------------
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(first_layer_hidden_nodes,
second_layer_hidden_nodes)
visible_biases = np.zeros(shape=(first_layer_hidden_nodes))
hidden_biases = np.zeros(shape=(second_layer_hidden_nodes))
# initialize Restricted Boltzman Machine
second_bbrbm = BBRBM(
n_visible=first_layer_hidden_nodes,
n_hidden=second_layer_hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
second_bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
second_bbrbm.fit(data_x=first_output,
validation_data_x=first_test_output,
n_epoches=epoches,
batch_size=batch_size)
rbm_second_layer_weight_matrix = second_bbrbm.get_weight_matrix()
rbm_second_layer_hidden_layer_biases = second_bbrbm.get_hidden_biases()
second_output = np.dot(first_output, rbm_second_layer_weight_matrix)
for i in range(second_output.shape[0]):
second_output[i, :] += rbm_second_layer_hidden_layer_biases
for j in range(second_output.shape[1]):
# sigmoid(relu(x))
second_output[i, j] = sigmoid(relu(second_output[i, j]))
second_test_output = np.dot(first_test_output,
rbm_second_layer_weight_matrix)
for i in range(second_test_output.shape[0]):
second_test_output[i, :] += rbm_second_layer_hidden_layer_biases
for j in range(second_test_output.shape[1]):
# sigmoid(relu(x))
second_test_output[i, j] = sigmoid(relu(second_test_output[i, j]))
# ---------------------- THIRD LAYER --------------------
if third_layer_hidden_nodes > 0:
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(second_layer_hidden_nodes,
third_layer_hidden_nodes)
visible_biases = np.zeros(shape=(second_layer_hidden_nodes))
hidden_biases = np.zeros(shape=(third_layer_hidden_nodes))
# initialize Restricted Boltzman Machine
third_bbrbm = BBRBM(
n_visible=second_layer_hidden_nodes,
n_hidden=third_layer_hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
third_bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
third_bbrbm.fit(data_x=second_output,
validation_data_x=second_test_output,
n_epoches=epoches,
batch_size=batch_size)
rbm_third_layer_weight_matrix = third_bbrbm.get_weight_matrix()
rbm_third_layer_hidden_layer_biases = third_bbrbm.get_hidden_biases()
third_output = np.dot(second_output, rbm_third_layer_weight_matrix)
for i in range(third_output.shape[0]):
third_output[i, :] += rbm_third_layer_hidden_layer_biases
for j in range(third_output.shape[1]):
# sigmoid(relu(x))
third_output[i, j] = sigmoid(relu(third_output[i, j]))
third_test_output = np.dot(second_test_output,
rbm_third_layer_weight_matrix)
for i in range(third_test_output.shape[0]):
third_test_output[i, :] += rbm_third_layer_hidden_layer_biases
for j in range(third_test_output.shape[1]):
# sigmoid(relu(x))
third_test_output[i, j] = sigmoid(relu(third_test_output[i,
j]))
# ------------------------ CLASSIFICATION ---------------------
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=third_output.shape[1],
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(
mean=0.0, stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(third_output,
np.asarray(y_train),
batch_size=batch_size,
epochs=epoches,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(third_test_output, np.asarray(y_test)))
score = model.evaluate(third_test_output,
np.asarray(y_test),
verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
return score[1]
else:
# ------------------------ CLASSIFICATION ---------------------
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=second_output.shape[1],
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(
mean=0.0, stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(second_output,
np.asarray(y_train),
batch_size=batch_size,
epochs=epoches,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(second_test_output, np.asarray(y_test)))
score = model.evaluate(second_test_output,
np.asarray(y_test),
verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
return score[1]
gbrbm.save_weights(filename=gb_filename, name=gb_name)
#begin pretraining the first Bernoulli-Bernoulli RBM
bb_input_data_1 = gbrbm.transform(input_data)
bb_input_data_1 = np.array(bb_input_data_1)
#print(bb_input_data_1.shape)
bb_n_visible_1 = bb_input_data_1.shape[1]
bb_n_hid_1 = 2048
bb_learning_rate_1 = 0.01
bb_momentum_1 = 0.95
bb_err_function_1 = 'mse'
bbrbm_1 = BBRBM(n_visible=bb_n_visible_1,
n_hidden=bb_n_hid_1,
learning_rate=bb_learning_rate_1,
momentum=bb_momentum_1,
err_function=bb_err_function_1,
use_tqdm=False)
bb_n_epoches_1 = 10
bb_batch_size_1 = 128
errs_1 = bbrbm_1.fit(data_x=bb_input_data_1,
n_epoches=bb_n_epoches_1,
batch_size=bb_batch_size_1,
shuffle=True,
verbose=True)
bb_filename_1 = 'pretrain_models/bbrbm_1.ckpt'
bb_name_1 = 'rbm'
bbrbm_1.save_weights(filename=bb_filename_1, name=bb_name_1)
def main():
logging.basicConfig(level=logging.INFO)
(x_train, _), (x_test, _) = tf.keras.datasets.mnist.load_data()
x_train = x_train / 255
x_test = x_test / 255
dataset = tf.data.Dataset.from_tensor_slices(
x_train.reshape(-1, 28 * 28).astype(np.float32))
dataset = dataset.shuffle(1024, reshuffle_each_iteration=True)
rbm_1 = BBRBM(n_visible=28 * 28, n_hidden=64)
rbm_2 = BBRBM(n_visible=64, n_hidden=32)
rbm_3 = BBRBM(n_visible=32, n_hidden=2)
epoches = 100
batch_size = 10
rbm_1.fit(dataset, epoches=epoches, batch_size=batch_size)
dataset_2 = transform_dataset(rbm_1, dataset)
rbm_2.fit(dataset_2, epoches=epoches, batch_size=batch_size)
dataset_3 = transform_dataset(rbm_2, dataset_2)
rbm_3.fit(dataset_3, epoches=epoches, batch_size=batch_size)
def encode(x):
hidden_1 = rbm_1.compute_hidden(x)
hidden_2 = rbm_2.compute_hidden(hidden_1)
hidden_3 = rbm_3.compute_hidden(hidden_2)
return hidden_3
dataset_test = tf.data.Dataset.from_tensor_slices(
x_test.reshape(-1, 28 * 28).astype(np.float32))
encoded_test = []
for batch in dataset_test.batch(2048):
batch_encoded_tensor = encode(batch)
batch_encoded = batch_encoded_tensor.numpy()
encoded_test.append(batch_encoded)
encoded_test = np.vstack(encoded_test)
plt.scatter(encoded_test[:, 0], encoded_test[:, 1], alpha=0.5)
plt.savefig('example.png')
#test to see if corretly added labels
# create figure
fig = plt.figure(figsize=(10, 10))
# setting values to rows and column variables
rows = 2
columns = 1
fig.add_subplot(rows, columns, 2)
plt.imshow(mnist_labeled[6][10:794].reshape(28,28))#new_image.reshape(29,29)
fig.add_subplot(rows, columns, 1)
plt.imshow(mnist_labeled[6][0:10].reshape(1,-1))
plt.locator_params(axis="x", nbins=9)
plt.show()
#for m in range(19):
# show_digit(mnist_labeled[m].reshape(28, 28))
ep = np.array([3000,4000,5000,6000,7000,8000,9000,10000,11000,12000,13000,15000,16000,17000,18000,19000,20000])#700, 800, 900, 1000, 1100, 1200,1300,1400, 1500,1600,1700]
bbrbm = BBRBM(n_visible=794, n_hidden=64, learning_rate=0.008, momentum=0.95, use_tqdm=True)
print("learing rate is ", 0.008)
#for i in range(ep.size):
err = bbrbm.fit(mnist_labeled, n_epoches=20001, batch_size=10)
#save the weights
#filename = fname[i] #'weights_class600ep'
#name1 = name[i] #'bbrbm_class600ep'
#bbrbm.save_weights(filename,name1)
#weights_class600ep = lr = 0.01 X
#weights_class600ep005 = lr = 0.005 X
#weights_class600ep002 = lr = 0.002 X
#weights_class600ep = lr = 0.008 X
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 1 14:41:33 2017
@author: rahul.ghosh
"""
import numpy as np
import matplotlib.pyplot as plt
from tfrbm import BBRBM, GBRBM
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)
mnist_images = mnist.train.images
bbrbm = BBRBM(n_visible=784, n_hidden=64, learning_rate=0.01, momentum=0.95, use_tqdm=True)
errs = bbrbm.fit(mnist_images, n_epoches=30, batch_size=10)
plt.plot(errs)
plt.show()
#Hi there, Can you see this
import numpy as np
import matplotlib.pyplot as plt
import os, sys, inspect
sys.path.insert(1, os.path.join(sys.path[0], '..'))
from tfrbm import BBRBM, GBRBM
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)
mnist_images = mnist.train.images
X_train = np.where(mnist_images > 0, 1, 0) #binrize the pixels
bbrbm = BBRBM(n_visible=784,
n_hidden=64,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
errs = bbrbm.fit(X_train, n_epoches=50, batch_size=10)
plt.plot(errs)
plt.show()
############################
#Test the Reconstruction of the RBM
IMAGE = 1
image = mnist_images[IMAGE]
#crop the imag
#crop dimentions
x = 6
y = 6
a = image.reshape(28, 28)
img = a[0:16, 0:28] #crop the image
for i in range(19):
b = np.pad(mnist_images1[i].reshape(28, 28), 1, pad_with, padder=0)#add contour of zeros around the image
x = np.where(mnist.train.labels[i] == 1)
#show_digit(mnist_images1[2].reshape(28, 28))
x1 = list(x) # change to list to add 20 offset
x1[0] = x1[0] +20
x =tuple(x1) #revert back to tuple
# print("x is", x[0])
# x[0] = x[0]+[20]
b[x,0] = 1 # add the label coressponding to the image on the right most clolumn [0..10] ,+20 means that we want to have the labels on pixels [20...29]
mnist_labeled[i] = b.flatten() #convert the image to 1D and store it
#test to see if corretly added labels
show_digit(mnist_labeled[5].reshape(30,30))#new_image.reshape(29,29)
for m in range(19):
show_digit(mnist_labeled[20].reshape(30, 30))
#create the BM
bbrbm = BBRBM(n_visible=900, n_hidden=64, learning_rate=0.01, momentum=0.95, use_tqdm=True)
err = bbrbm.fit(mnist_labeled, n_epoches=100000, batch_size=10)
#save the weights
filename = 'weights_class'
name = 'bbrbm_class'
bbrbm.save_weights(filename,name)
def rbm(hidden_nodes, learning_rate, epoches, batch_size=128):
# ---------------------- PRE-TRAINING --------------------
# gets datasets: x_{train,test} are np arrays
# y_{train,test} are categorical dstributions
# over a digit
x_train, x_test, y_train, y_test = get_datasets()
# Generate weights ~ N(mean =0, stdev =0.01) & zero biases
weight_matrix = generate_weight_matrix(x_train.shape[1], hidden_nodes)
visible_biases = np.zeros(shape=(x_train.shape[1]))
hidden_biases = np.zeros(shape=(hidden_nodes))
# initialize Restricted Boltzman Machine
bbrbm = BBRBM(
n_visible=x_train.shape[1],
n_hidden=hidden_nodes,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
# fit model
bbrbm.fit(data_x=x_train,
validation_data_x=x_test,
n_epoches=epoches,
batch_size=batch_size)
rbm_weight_matrix = bbrbm.get_weight_matrix()
rbm_hidden_layer_biases = bbrbm.get_hidden_biases()
# TODO: Import commands for comparative representation of
# TODO: original input vs reconstruction input
# generate output that will be used as input in the classification stage
output = np.dot(x_train, rbm_weight_matrix)
for i in range(output.shape[0]):
output[i, :] += rbm_hidden_layer_biases
for j in range(output.shape[1]):
# my_sigmoid(relu(x))
output[i, j] = my_sigmoid(relu(output[i, j]))
test_output = np.dot(x_test, rbm_weight_matrix)
for i in range(test_output.shape[0]):
test_output[i, :] += rbm_hidden_layer_biases
for j in range(test_output.shape[1]):
# my_sigmoid(relu(x))
test_output[i, j] = my_sigmoid(relu(test_output[i, j]))
# ------------------------ CLASSIFICATION ---------------------
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=output.shape[1],
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(output,
np.asarray(y_train),
batch_size=batch_size,
epochs=epoches,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(test_output, y_test))
score = model.evaluate(test_output, np.asarray(y_test), verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
return score[1]
abnormal_code = "11"
train_data_list, label_list = word2vecCNN.embedding_data(abnormal_code)
train_x = np.reshape(train_data_list, (len(train_data_list), -1))[:10]
train_y = np.array(label_list)[:10]
print("train_data_list.shape", train_x.shape)
print("label_list.shape", train_y.shape)
test_x = train_x[7:]
test_y = train_y[7:]
print("test_data_list.shape", test_x.shape)
print("label_list.shape", test_y.shape)
# RBM
bbrbm = BBRBM(n_visible=train_x.shape[1],
n_hidden=640,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
errs = bbrbm.fit(train_x, n_epoches=2, batch_size=10)
doc_vec = bbrbm.transform(train_x.reshape(train_x.shape[0], -1))
# doc_vec = doc_vec.reshape(train_data_list.shape[0],10,-1)
print("doc_vec.shape", doc_vec.shape)
# tensorflow placeholders
tf_x = tf.placeholder(
tf.float32, [None, TIME_STEP * INPUT_SIZE]) # shape(batch, 64*10)
image = tf.reshape(
tf_x, [-1, TIME_STEP, INPUT_SIZE]) # (batch, height, width, channel)
tf_y = tf.placeholder(tf.int32, [None, 2]) # input y
# RNN
weights = []
biases = []
with tf.device('/gpu:%d' % gpu_device):
for i in range(rbm_hidden_num):
# 训练rbm
if i == 0 and rbm_type == 'GBRBM':
rbm = GBRBM(n_visible=rbm_visible_size,
n_hidden=rbm_hidden_size,
learning_rate=0.01,
momentum=0.95,
use_tqdm=False)
else:
rbm = BBRBM(n_visible=rbm_visible_size,
n_hidden=rbm_hidden_size,
learning_rate=0.01,
momentum=0.95,
use_tqdm=False)
errs = rbm.fit(rbm_x,
n_epoches=10,
batch_size=100,
verbose=True)
rbm_x = rbm.transform(rbm_x)
rbm_W, vb, rbm_b = rbm.get_weights()
rbm_visible_size = rbm_hidden_size
weights.append(rbm_W)
biases.append(rbm_b)
with tf.name_scope('dbn'):
dbn_x = tf.placeholder(tf.float32,
[None, time_step * input_size],
# cv2.imshow('image',image)
# cv2.waitKey(0)
Images.append(image)
Images = np.array(Images, np.int64)
print(Images.shape)
print(Images.dtype)
# Images=Images[:,:,:,np.newaxis]
return Images
if __name__ == '__main__':
Images = get_data(
'/home/cooper/PycharmProjects/zuoye/math_theory_work/作业2/images')
mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)
mnist_images = mnist.train.images
bbrbm = BBRBM(n_visible=914 * 990 / 4,
n_hidden=500,
learning_rate=0.001,
momentum=0.95,
use_tqdm=True)
errs = bbrbm.fit(Images, n_epoches=30, batch_size=10)
plt.plot(errs)
# plt.show()
# model=Get_model()
# model.compile(optimizer='sgd',loss=sparse_categorical_crossentropy,metrics=['accuracy'])
# model.fit(Images,Images,batch_size=1,epochs=10,validation_split=0.2,callbacks=[TensorBoard('/home/cooper/PycharmProjects/zuoye/math_theory_work/logs')])
#
def run(first_hidden, second_hidden, learning_rate, epochs, batch_size):
x_train, x_test, y_train, y_test = get_datasets()
# ------------------FIRST HIDDEN-------------------
weight_matrix = generate_weight_matrix(x_train.shape[1], hidden)
visible_biases = np.zeros(shape=(784))
hidden_biases = np.zeros(shape=(first_hidden))
bbrbm = BBRBM(
n_visible=784,
n_hidden=first_hidden,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
bbrbm.set_weights(weight_matrix, visible_biases, hidden_biases)
errs = bbrbm.fit(x_train, n_epoches=epochs, batch_size=batch_size)
first_weight_matrix = np.asarray(bbrbm.get_weight_matrix())
first_visible_biases = bbrbm.get_visible_biases()
first_hidden_biases = bbrbm.get_hidden_biases()
first_output = np.dot(x_test, weight_matrix)
for i in range(first_output.shape[0]):
first_output[i, :] += first_hidden_biases
for j in range(first_output.shape[1]):
# sigmoid(relu(x))
first_output[i, j] = sigmoid(np.max(0, first_output[i, j]))
# ------------------SECOND HIDDEN-------------------
bbrbm_second = BBRBM(
n_visible=first_hidden,
n_hidden=784,
learning_rate=learning_rate,
momentum=0,
# momentum=0.95,
use_tqdm=True)
trained_outputs = np.ndarray(shape=(x_train.shape[0], x_train.shape[1]))
for i in range(trained_outputs.shape[0]):
trained_outputs[i, :] = bbrbm.reconstruct(x_train[i].reshape(1, -1))
sgd = optimizers.SGD(lr=0.3, decay=0, momentum=0, nesterov=True)
model = Sequential()
model.add(
Dense(10,
input_dim=784,
activation="softmax",
kernel_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.01),
bias_initializer='zeros'))
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
earlyStopping = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
model.fit(trained_outputs,
np.asarray(y_train),
batch_size=batch_size,
epochs=epochs,
verbose=1,
shuffle=True,
callbacks=[earlyStopping],
validation_data=(x_test, y_test))
score = model.evaluate(x_test, np.asarray(y_test), verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
# vis(x_train, trained_outputs, 20)
trained_outputs = np.dot(x_train, weight_matrix)
for i in range(trained_outputs.shape[0]):
trained_outputs[i, :] += hidden_biases
debug = 0
# test = np.dot(x_test, weight_matrix)
#
# preds = []
# for i in range(test.shape[0]):
#
# test[i,:]+=hidden_biases
# output_weight_matrix = generate_output_weight_matrix(test.shape[1])
# sgd = optimizers.SGD(lr=0.01, decay=0, momentum=0, nesterov=True)
# model = Model(inputs=test, outputs=y_test)
model.compile(optimizer=sgd,
loss='mean_squared_error',
metrics=['accuracy'],
loss_weights=None,
sample_weight_mode=None,
weighted_metrics=None,
target_tensors=None)
debug = 0
# fix ssl error when downloading files
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
# %%
# read mnist data
mnist = input_data.read_data_sets('../../data/MNIST', one_hot=True)
mnist_train_images = mnist.train.images
mnist_test_images = mnist.test.images
mnist_test_labels = mnist.test.labels
# %%
# fit Bernoulli-Bernoulli RBM model
bbrbm = BBRBM(n_visible=784,
n_hidden=64,
learning_rate=0.01,
momentum=0.95,
use_tqdm=True)
bbrbm_errs = bbrbm.fit(mnist_train_images, n_epoches=30, batch_size=10)
# %%
# sava trained RBM model
bbrbm.save_weights('../../output/cn/2018-01-17-ising-hopfield-and-rbm/mnist-bbrbm', 'default')
# bbrbm.load_weights('../../output/cn/2018-01-17-ising-hopfield-and-rbm/mnist-bbrbm', 'default')
# %%
# plot bbrbm errors
plt.style.use('ggplot')
plt.plot(bbrbm_errs)
plt.savefig('../../static/cn/2018-01-17-ising-hopfield-and-rbm/bbrbm-mnist-errs.png', bbox_inches='tight')
import matplotlib.pyplot as plt
from tfrbm import BBRBM, GBRBM, RegRBM
data = np.array([[1, 1, 0], [0, 1, 1]], float)
num_sim = 5
for x in range(num_sim):
for std in [0.1, 0.2]:
print('###' + str(x) + '###')
print('Xavier weight init')
bbrbm_xavier = BBRBM(n_visible=3,
n_hidden=2,
learning_rate=0.05,
momentum=0,
use_tqdm=False,
init_weight_scheme='xavier',
rbmName='Xavier weight init',
stddev_par=std)
# print(bbrbm_xavier.get_weights())
errs = bbrbm_xavier.fit(data, n_epoches=400, batch_size=1)
# plt.plot(errs)
# plt.show()
# print(bbrbm_xavier.get_weights())
bbrbm_xavier.export_to_csv()
# print('Uniform one weight init')
# bbrbm_ones = BBRBM(n_visible=3, n_hidden=2, learning_rate=0.05, momentum=0, use_tqdm=False, init_weight_scheme ='ones', rbmName = 'Uniform one weight init')
# # print(bbrbm_ones.get_weights())
# errs = bbrbm_ones.fit(data, n_epoches=400, batch_size=1)
# # plt.plot(errs)