from rbm import RestrictedBoltzmannMachine
from dbn import DeepBeliefNet
import matplotlib.pyplot as plt
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(
dim=image_size, n_train=60000, n_test=10000
)
dbn = DeepBeliefNet(
sizes={
"vis": image_size[0] * image_size[1],
"hid": 500,
"pen": 500,
"top": 2000,
"lbl": 10,
},
image_size=image_size,
n_labels=10,
batch_size=20,
)
dbn.train_greedylayerwise(
vis_trainset=train_imgs, lbl_trainset=train_lbls, n_iterations=10
)
rbm0 = dbn.rbm_stack["vis--hid"]
rbm1 = dbn.rbm_stack["hid--pen"]
rbm2 = dbn.rbm_stack["pen+lbl--top"]
image0 = train_imgs[:1,:]
rbm.cd1(X=train_imgs, n_iterations=30000)
if DBN:
# Deep Belief Net
if TWO_LAYER:
print ("\n>>> Starting a two-layer Deep Belief Net...")
# Initialize the two layer network
dbn = DeepBeliefNetTwoLayer(sizes={"vis": image_size[0] *
image_size[1], "hid": 500, "top": 2000, "lbl": 10},
image_size=image_size, n_labels=10, batch_size=10)
else:
print ("\n>>> Starting a two-layer Deep Belief Net...")
# Initialize the three layer network
dbn = DeepBeliefNet(sizes={"vis": image_size[0] * image_size[1],
"hid": 500, "pen": 500, "top": 2000, "lbl": 10},
image_size=image_size, n_labels=10, batch_size=10)
# Greedy layer-wise training
dbn.train_greedylayerwise(X=train_imgs, y=train_lbls,
n_iterations=60000, compute_rec_err=True)
if CLASSIFY:
# Recognize/Classify images
dbn.recognize(train_imgs, train_lbls)
dbn.recognize(test_imgs, test_lbls)
if GENERATE:
# Generate prototyical digits
for digit in range(10):
digit_1hot = np.zeros(shape=(1, 10))
# )
# rbm.cd1(visible_trainset=train_imgs, n_iterations=10)
# save_stuff(f"savefiles/{n_hidden}_hidden_trained_params.npz", [rbm.weight_vh, rbm.bias_h, rbm.bias_v])
# exit()
""" deep- belief net """
print("\nStarting a Deep Belief Net..")
dbn = DeepBeliefNet(
sizes={
"vis": image_size[0] * image_size[1],
"hid": 500,
"pen": 500,
"top": 2000,
"lbl": 10,
},
image_size=image_size,
n_labels=10,
batch_size=200,
)
""" greedy layer-wise training """
dbn.train_greedylayerwise(vis_trainset=train_imgs,
lbl_trainset=train_lbls,
n_iterations=10)
# dbn.recognize(train_imgs, train_lbls)
#for digit in range(1,10):
# digit_1hot = np.zeros(shape=(1, 10))
img_num = 60000
test_num = 10000
layers = 3
components = [1000, 1000, 1000]
batch_size = 50
learning_rate = 0.1
bias_learning_rate = 0.1
epochs = 100
sparsity = [0.01, None, None]
train_img = mndata.train_images[:img_num] > 0.5
train_lab = mndata.train_labels[:img_num]
test_img = mndata.test_images[:test_num] > 0.5
test_lab = mndata.test_labels[:test_num]
DBN = DeepBeliefNet(layers, components, batch_size, learning_rate, bias_learning_rate, epochs, sparsity_rate=sparsity)
DBN.plot_weights = False
DBN.plot_histograms = True
DBN.fit_network(train_img, train_lab)
## the label column will get roc and pr curves relatives to that column (one versus all). Here we look at the number 1
results = DBN.results(test_img, test_lab, label_column=1, write_file=os.path.join(repo_dir, "examples/results/results.json"))
pyplot.close()
df = pandas.DataFrame({"true positive rates": results['tpr'], "false positive rates": results['fpr']})
df.plot(x="false positive rates", y="true positive rates", figsize=(10, 6), linewidth=2)
pyplot.plot([0, 1], [0, 1], linestyle='--')
pyplot.title("AUC = %0.3f" % results['auc'])
pyplot.savefig(os.path.join(repo_dir, "examples/results/roc.svg"))
from dbn import DeepBeliefNet
if __name__ == "__main__":
image_size = [28,28]
train_imgs,train_lbls,test_imgs,test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000)
''' deep- belief net '''
print ("\nStarting a Deep Belief Net..")
batch_size = 20
dbn = DeepBeliefNet(sizes={"vis":image_size[0]*image_size[1], "hid":500, "pen":500, "top":2000, "lbl":10},
image_size=image_size,
n_labels=10,
batch_size=batch_size
)
''' greedy layer-wise training '''
epochs = 10
iterations = int(60000*epochs/batch_size)
dbn.train_greedylayerwise(vis_trainset=train_imgs, lbl_trainset=train_lbls, n_iterations=iterations)
#dbn.recognize(train_imgs, train_lbls)
dbn.recognize(test_imgs, test_lbls)
for digit in range(10):
plt.plot(range(1,11), er[:,2],'g',label='400 nodes hidden layer')
plt.plot(range(1,11), er[:,3],'r',label='500 nodes hidden layer')
plt.xlabel('Epochs')
plt.ylabel('Reconstruction loss')
plt.title('Learning curve')
plt.legend()
#%%
print ("\nStarting a Deep Belief Net..")
dbn = DeepBeliefNet(sizes={"vis":image_size[0]*image_size[1], "hid":500, "pen":500, "top":2000, "lbl":10},
image_size=image_size,
n_labels=10,
batch_size=10
)
''' greedy layer-wise training '''
dbn.train_greedylayerwise(vis_trainset=train_imgs, lbl_trainset=train_lbls, n_iterations=1)
dbn.recognize(train_imgs, train_lbls)
dbn.recognize(test_imgs, test_lbls)
#%%
dbn = DeepBeliefNet(sizes={"vis":image_size[0]*image_size[1], "hid":500, "pen":500, "top":2000, "lbl":10},
image_size=image_size,
# image_size=image_size,
# is_top=False,
# n_labels=10,
# batch_size=20
# )
# rbm.cd1(visible_trainset=train_imgs, n_iterations=20)
''' deep- belief net '''
print("\nStarting a Deep Belief Net..")
dbn = DeepBeliefNet(sizes={
"vis": image_size[0] * image_size[1],
"hid": 500,
"pen": 500,
"top": 2000,
"lbl": 10
},
image_size=image_size,
n_labels=10,
batch_size=100)
''' greedy layer-wise training '''
dbn.train_greedylayerwise(vis_trainset=train_imgs,
lbl_trainset=train_lbls,
n_iterations=20,
verbose=True)
# print("Classify training set")
# dbn.recognize(train_imgs, train_lbls)
#
# print("Classify test set")
# for i, hidden in enumerate([200, 500]):
# plt.plot(range(10, 21), averages[i], label=str(hidden) + " hidden units")
# plt.xticks(range(10, 21))
# plt.xlabel("Epoch")
# plt.ylabel("Average Loss rate")
# plt.legend()
# plt.show()
''' deep-belief net '''
print("\nStarting a Deep Belief Net..")
dbn = DeepBeliefNet(sizes={
"vis": image_size[0] * image_size[1],
"hid": 500,
"pen": 500,
"top": 2000,
"lbl": 10
},
image_size=image_size,
n_labels=10,
batch_size=20)
''' greedy layer-wise training '''
aux = dbn.train_greedylayerwise(vis_trainset=train_imgs,
lbl_trainset=train_lbls,
n_iterations=ITERATIONS)
# plt.plot(range(10, 21), averages[-1], label="Original RBM")
# plt.plot(range(10, 21), aux, label="Greedy RBM")
# plt.xticks(range(10, 21))
# plt.xlabel("Epoch")
# plt.ylabel("Average Loss rate")
# plt.legend()
# image_size=image_size,
# is_top=False,
# n_labels=10,
# batch_size=20
# )
# rbm.cd1(visible_trainset=train_imgs, n_iterations=18000)
''' deep- belief net '''
print("\nStarting a Deep Belief Net..")
dbn = DeepBeliefNet(sizes={
"vis": image_size[0] * image_size[1],
"hid": 500,
"pen": 500,
"top": 2000,
"lbl": 10
},
image_size=image_size,
n_labels=10,
batch_size=20)
''' greedy layer-wise training '''
dbn.train_greedylayerwise(vis_trainset=train_imgs,
lbl_trainset=train_lbls,
n_iterations=90000)
# dbn.recognize(train_imgs[:1000, :], train_lbls[:1000, :])
# dbn.recognize(test_imgs, test_lbls)
imgs = []
