def testing_run_hardcoded_RBM_pass(self):
self.rbms.append(
rbm.RBM(NUM_DATAPOINTS,
3,
visible_unit_type='gauss',
model_name=self.name + "_layer_1",
verbose=1,
main_dir='layer_1_test'))
self.rbms.append(
rbm.RBM(3,
2,
visible_unit_type='bin',
model_name=self.name + '_layer_2',
verbose=1,
main_dir='layer_2_test'))
training_set = pd.read_csv(TRAINING_SET_PATH, sep=',', header=None)
training_set = training_set.values
self.rbms[0].fit(training_set)
test_set = pd.read_csv(TEST_SET_PATH, sep=',', header=None)
test_set = test_set.values
training_set_transform = self.rbms[0].transform(training_set)
testing_tools.write_csv(
training_set_transform,
TESTING_PATH + '/L1_training_set_transformed.csv')
self.rbms[1].fit(training_set_transform)
x = self.rbms[1].transform(training_set_transform)
testing_tools.write_csv(
x, TESTING_PATH + '/L2_training_set_transformed_transformed.csv')
def _main():
data, _ = mnist.MNIST("train",
path="../../machine-learning/data/mnist/",
data_size=40,
batch_size=20,
reshape=False,
one_hot=False,
binarize=True).to_ndarray()
test_data, _ = mnist.MNIST("test",
path="../../machine-learning/data/mnist/",
data_size=40,
batch_size=20,
reshape=False,
one_hot=False,
binarize=True).to_ndarray()
max_epoch = 1
# Layer 1
print("----- Layer 1 -----")
layer_i = rbm.RBM(train_data=data, num_hidden=1000)
layer_i.train(max_epoch=max_epoch)
# layer_i_param = (layer_i.weight, layer_i.visible_bias, layer_i.hidden_bias)
# Layer 2
print("----- Layer 2 -----")
layer_ii = rbm.RBM(train_data=layer_i.hidden_data, num_hidden=500)
layer_ii.train(max_epoch=max_epoch)
# layer_ii_param = (layer_ii.weight, layer_ii.visible_bias, layer_ii.hidden_bias)
# Layer 3
print("----- Layer 3 -----")
layer_iii = rbm.RBM(train_data=layer_ii.hidden_data, num_hidden=250)
layer_iii.train(max_epoch=max_epoch)
# layer_iii_param = (layer_iii.weight, layer_iii.visible_bias, layer_iii.hidden_bias)
# Layer 4
print("----- Layer 4 -----")
layer_iv = rbm.RBM(train_data=layer_iii.hidden_data, num_hidden=30)
layer_iv.train(max_epoch=max_epoch)
# layer_iv_param = (layer_iv.weight, layer_iv.visible_bias, layer_iv.hidden_bias)
# Backpropagation
print("\n=============== Backpropagation ===============\n")
bp.backpropagation(layers=[layer_i, layer_ii, layer_iii, layer_iv],
train_data=data,
test_data=test_data,
max_epoch=2)
def RBM1(X_train, X_test, y_train, y_test):
n = X_train.to_numpy()
g1 = rbm.RBM(num_visible=1034, num_hidden=500)
g1.train(n)
user = X_test.to_numpy()
#print(np.array([X_train.loc[2732].to_numpy()]))
prediction = pd.DataFrame(g1.run_visible(user))
def main(dirname, num_hidden):
"""
Main program for running the restricted Boltzmann machine
:param dirname: string with name of the directory in which the input files are present
:param num_hidden: integer corresponding with the number of hidden nodes
"""
DATA_SPLIT = 0.8
# Select a range of number of hidden nodes, which needs to be optimized
# num_hidden = np.array(range(100, 50, 200))
# filepath = "./binary/"
filepath = "./"
# training_data = hf.load_hdf5(filepath + dirname + "/brain_data_set.hdf5")
# training_data = pd.DataFrame(pd.read_csv("./boltzmann_machine_toy_data.csv", ',')).as_matrix()
training_data = hf.load_hdf5(filepath + dirname + ".hdf5")
print("Finished loading data. Start training")
r = rbm.RBM(training_data=training_data,
num_visible=training_data.shape[1],
num_hidden=int(num_hidden))
# r = rbm.RBM(training_data=training_data, num_visible=training_data.shape[1], num_hidden=int(num_hidden))
r.train(outfile=dirname,
split=DATA_SPLIT,
max_iterations=1000,
lr=0.1,
k=1,
visualize=False)
# r.test(split=DATA_SPLIT)
r.final_hid_recon()
r.save_parameters(dirname) # Save output
def setUp(self):
self.rbm = rbm.RBM(3, 2)
self.rbm.weights = np.array([[0.778157, 0.118274, 0.264556],
[0.870012, 0.639921, 0.774234]])
self.visible_layer = np.array([[1, 1, 1], [1, 0, 1], [1, 0, 0],
[0, 0, 1]])
self.hidden_layer = np.array([[1, 1], [0, 1], [0, 1], [0, 0]])
def __init__(self, numNodes, batchSize=100, actType='Logistic'):
self.numNodes = numNodes
self.actType = actType
self.batchSize = batchSize
self.rbms = [
rbm.RBM(numNodes[i],
numNodes[i + 1],
batchSize=self.batchSize,
actType=self.actType) for i in range(len(numNodes) - 1)
]
def __init__(self,
numpt_rng,
theano_rng=None,
n_in=784,
hidden_layers_size=[500, 500],
n_out=10):
self.sigmodi_layers = []
self.rbm_layers = []
self.params = []
self.n_layers = len(hidden_layers_size)
assert self.n_layers >= 0
if not theano_rng:
theano_rng = MRG_RandomStreams(numpt_rng.randint(2**30))
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in xrange(self.n_layers):
if i == 0:
input_size = n_in
else:
input_size = hidden_layers_size[i - 1]
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmodi_layers[-1].output
sigmoid_layer = logistic.HiddenLayer(
rng=numpt_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_size[i],
)
self.sigmodi_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
rbm_layer = rbm.RBM(inputs=layer_input,
n_visiable=input_size,
n_hidden=hidden_layers_size[i],
numpy_rng=numpt_rng,
theano_rng=theano_rng,
W=sigmoid_layer.W,
h_bias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
self.logLayer = logistic.Softmax_layer(
inputs=self.sigmodi_layers[-1].output,
n_in=hidden_layers_size[-1],
n_out=n_out,
)
self.params.extend(self.logLayer.params)
self.finetune_cost = self.logLayer.nagetive_likehood(self.y)
self.errors = self.logLayer.error(self.y)
def __init__(self, input_size, dbn_shape):
self._rbm_hidden_sizes = dbn_shape
self._input_size = input_size
self.rbm_list = []
print("Construct DBN ...")
for i, size in enumerate(dbn_shape):
print('RBM {} : {} -> {}'.format(i, input_size, size))
self.rbm_list.append(rbm.RBM(input_size, size))
input_size = size
def conditioned_RBM(RBM, conditions):
num_conditions = len(conditions)
l_hh = np.array([condition[0] for condition in conditions])
l_value = np.array([condition[1] for condition in conditions])
remaining = np.array([x for x in range(RBM.n_h) if not x in l_hh])
tmp_RBM = rbm.RBM(n_v=RBM.n_v,
n_h=RBM.n_h - num_conditions,
n_cv=RBM.n_cv,
n_ch=RBM.n_ch,
visible=RBM.visible,
hidden=RBM.hidden)
tmp_RBM.vlayer.fields = RBM.vlayer.fields.copy()
tmp_RBM.vlayer.fields0 = RBM.vlayer.fields0.copy()
tmp_RBM.weights = RBM.weights[remaining, :]
if RBM.hidden in ['Bernoulli', 'Spin']:
tmp_RBM.hlayer.fields = RBM.hlayer.fields[remaining].copy()
tmp_RBM.hlayer.fields0 = RBM.hlayer.fields0[remaining].copy()
elif RBM.hidden == 'Gaussian':
tmp_RBM.hlayer.a = RBM.hlayer.a[remaining].copy()
tmp_RBM.hlayer.a0 = RBM.hlayer.a0[remaining].copy()
tmp_RBM.hlayer.b = RBM.hlayer.b[remaining].copy()
tmp_RBM.hlayer.b0 = RBM.hlayer.b0[remaining].copy()
elif RBM.hidden == 'ReLU':
tmp_RBM.hlayer.a = RBM.hlayer.a[remaining].copy()
tmp_RBM.hlayer.a0 = RBM.hlayer.a0[remaining].copy()
tmp_RBM.hlayer.theta_plus = RBM.hlayer.theta_plus[remaining].copy()
tmp_RBM.hlayer.theta_plus0 = RBM.hlayer.theta_plus0[remaining].copy()
tmp_RBM.hlayer.theta_minus = RBM.hlayer.theta_minus[remaining].copy()
tmp_RBM.hlayer.theta_minus0 = RBM.hlayer.theta_minus0[remaining].copy()
elif RBM.hidden == 'dReLU':
tmp_RBM.hlayer.a = RBM.hlayer.a[remaining].copy()
tmp_RBM.hlayer.a_plus = RBM.hlayer.a_plus[remaining].copy()
tmp_RBM.hlayer.a_plus0 = RBM.hlayer.a_plus0[remaining].copy()
tmp_RBM.hlayer.a_minus = RBM.hlayer.a_minus[remaining].copy()
tmp_RBM.hlayer.a_minus0 = RBM.hlayer.a_minus0[remaining].copy()
tmp_RBM.hlayer.theta_plus = RBM.hlayer.theta_plus[remaining].copy()
tmp_RBM.hlayer.theta_plus0 = RBM.hlayer.theta_plus0[remaining].copy()
tmp_RBM.hlayer.theta_minus = RBM.hlayer.theta_minus[remaining].copy()
tmp_RBM.hlayer.theta_minus0 = RBM.hlayer.theta_minus0[remaining].copy()
elif RBM.hidden == 'ReLU+':
tmp_RBM.hlayer.a = RBM.hlayer.a[remaining].copy()
tmp_RBM.hlayer.a0 = RBM.hlayer.a0[remaining].copy()
tmp_RBM.hlayer.theta_plus = RBM.hlayer.theta_plus[remaining].copy()
tmp_RBM.hlayer.theta_plus0 = RBM.hlayer.theta_plus0[remaining].copy()
tmp_RBM.vlayer.fields += (RBM.weights[l_hh] *
l_value[:, np.newaxis, np.newaxis]).sum(0)
tmp_RBM.vlayer.fields0 += (RBM.weights[l_hh] *
l_value[:, np.newaxis, np.newaxis]).sum(0)
return tmp_RBM
def __init__(self,
n_components=-1,
learning_rate=-1,
n_iter=-1,
num_visible=-1):
self.n_components = n_components
self.learning_rate = learning_rate
self.n_iter = n_iter
self.num_visible = num_visible
self.rbm_ = rbm.RBM(num_hidden=self.n_components,
learning_rate=self.learning_rate,
max_epochs=self.n_iter,
num_visible=num_visible)
def rbm_example():
digits = datasets.load_digits()
X = digits.images.reshape((digits.images.shape[0], -1))
X = (X / 16.0)
y = ut.all_to_sparse(digits.target, max(digits.target) + 1)
X, y, X_val, y_val, X_test, y_test = neur.cross_validation_sets(
np.array(X), np.array(y), "digits_rbm", True)
X_val = np.vstack([X_val, X_test])
y_val = np.vstack([y_val, y_test])
hid_layer = 300
bm = rbm.RBM(64, hid_layer)
#exit()
costs = bm.optimize(neur.mini_batch_generator(X), 2000, 0.08)
print "validate squared_error", bm.validate(X_val)
#exit()
filename = './random_set_cache/data_rbm_run.pkl'
first_layer_weights = np.hstack([np.zeros((hid_layer, 1)), bm.weights])
#pickle.dump(first_layer_weights, open(filename, 'w'))
# first_layer_weights = pickle.load(open(filename, 'r'))
thetas = neur.create_initial_thetas([64, hid_layer, 10], 0.12)
thetas[0] = first_layer_weights
thetas, costs, val_costs = neur.gradient_decent_gen(
izip(neur.mini_batch_generator(X, 10),
neur.mini_batch_generator(y, 10)),
learning_rate=0.05,
hidden_layer_sz=hid_layer,
iter=8000,
thetas=thetas,
X_val=X_val,
y_val=y_val,
do_early_stopping=True)
h_x, a = neur.forward_prop(X_test, thetas)
print "percentage correct predictions: ", ut.percent_equal(
ut.map_to_max_binary_result(h_x), y_test)
print "training error:", costs[-1:][0]
print "validation error:", val_costs[-1:][0]
print "lowest validation error:", min(val_costs)
plt.plot(costs, label='cost')
plt.plot(val_costs, label='val cost')
plt.legend()
plt.ylabel('error rate')
plt.show()
def train(self):
self.first_hidden_value=numpy.zeros((len(self.rates),self.net[1]))
for i in range(len(self.rates)):
#print "processing rating: " ,i
Wsel=self.W[0][self.itemids[i]:self.itemids[i]+self.k]
#print Wsel
self.first_rbm=rbm.RBM(numpy.array([self.rates[i]]), n_visible=self.k, n_hidden=self.net[1],
W=Wsel, hbias=None, vbias=None, numpy_rng=None)
self.first_rbm.contrastive_divergence(lr=0.001, k=5)
after_W, afterhbias = self.first_rbm.get_parameter()
self.W[0][self.itemids[i]:self.itemids[i]+self.k] = after_W
def __init__(self, net=[], k=5):
self.k=k
self.net=net
self.layers=len(net)
self.W=[]
self.hbias=[]
self.dbn_rbm=[]
for i in range(self.layers-1):
numpy_rng = numpy.random.RandomState(123+i)
a=1./net[i]
self.dbn_rbm.append(rbm.RBM(input=None, n_visible=net[i], n_hidden=net[i+1], W=None, hbias=None,
vbias=None, numpy_rng=None))
W , hbias =self.dbn_rbm[i].get_parameter()
self.W.append(W)
self.hbias.append(hbias)
def __init__(self, vsize=None, hsizes=[], lr=None, bsize=10, seed=123):
assert vsize and hsizes and lr
input = T.dmatrix('global_input')
self.layers = []
for hsize in hsizes:
r = rbm.RBM(input=input,
vsize=vsize,
hsize=hsize,
bsize=bsize,
lr=lr,
seed=seed)
self.layers.append(r)
# configure inputs for subsequent layer
input = self.layers[-1].hid
vsize = hsize
def __init__(self, layers, learning_rate):
assert(len(layers) == len(learning_rate) + 1)
self.rbms = []
for i,_ in enumerate(layers[:-1]):
r = rbm.RBM(layers[i], layers[i+1], learning_rate[i] )
self.rbms.append(r)
def arr_to_binary(num):
if num >= 0.5:
return 1.0
return 0.0
v = np.vectorize(arr_to_binary)
def to_b(self, data):
return v(data)
self.to_binary = v
def _create_RBMs(self):
rbm_id = 0
# for each shape in rbm_shapes, create a corresponding rbm.RBM and add it to the rbms list
for shape in self.rbm_shapes:
rbm_id = rbm_id + 1
rbm_name = self.uid + '_rbm_' + str(rbm_id)
visible, hidden = shape
vut = 'bin' # the visible unit type of all but first layer are of input layer binary, the first is gauss
if rbm_id == 1:
vut = 'gauss'
r = rbm.RBM(visible,
hidden,
visible_unit_type=vut,
model_name=rbm_name,
verbose=1,
main_dir=self.main_dir)
self.rbms.append(r)
def Weight_Matrix(size, ratio_hidden, epochs, training_data, type='1d'):
if type == '1d':
vis = size
hid = int(vis * ratio_hidden)
if type == '2d':
vis = size**2
if type == '3d':
vis = size**3
hid = int(vis * ratio_hidden)
r = rbm.RBM(num_visible=vis, num_hidden=hid)
r.train(training_data, epochs=epochs, learning_rate=0.1, batch_size=100)
weights = np.delete(np.delete(r.weights, 0, axis=0), 0, axis=1)
weights_hid = np.matmul(weights.T, weights)
X, Y = np.meshgrid(np.linspace(1, hid, hid), np.linspace(1, hid, hid))
z_min, z_max = -np.abs(weights_hid).max(), np.abs(weights_hid).max()
fig, ax = plt.subplots()
c = ax.pcolormesh(X, Y, weights_hid, cmap='RdBu', vmin=z_min, vmax=z_max)
ax.set_title('Correlation of Hidden activations to next Sample')
# set the limits of the plot to the limits of the data
ax.axis([X.min(), X.max(), Y.min(), Y.max()])
ax.set_xlabel('hidden index')
ax.set_ylabel('hidden index')
fig.colorbar(c, ax=ax)
weights_vis = np.matmul(weights, weights.T)
print(weights_vis)
X, Y = np.meshgrid(np.linspace(1, vis, vis), np.linspace(1, vis, vis))
#z_min, z_max = -np.abs(weights_vis).max(), np.abs(weights_vis).max()
z_min, z_max = weights_vis.min(), weights_vis.max()
fig, ax = plt.subplots()
c = ax.pcolormesh(X, Y, weights_vis, cmap='RdBu', vmin=z_min, vmax=z_max)
ax.set_title('Correlation of Visible activations to next Sample')
# set the limits of the plot to the limits of the data
ax.axis([X.min(), X.max(), Y.min(), Y.max()])
ax.set_xlabel('visible index')
ax.set_ylabel('visible index')
fig.colorbar(c, ax=ax)
def run():
print "Loading data..."
# load training data
trainImages, trainLabels = data_loader.load_mnist_train()
imDim = trainImages.shape[0]
visibleSize = 784 # 69
hiddenSize = 500
trainImages = trainImages.reshape(imDim**2, -1)
import numpy as np
trainImages = trainImages - np.mean(trainImages, axis=1).reshape(-1, 1)
# preprocess
print "Preprocessing Data..."
#prp = preprocess.Preprocess()
#prp.computePCA(trainImages)
# prp.plot_explained_var()
#trainImages = prp.whiten(trainImages,numComponents=visibleSize)
RBM = rbm.RBM(visibleSize,
hiddenSize,
grbm=True,
sp_target=0.05,
sp_weight=5)
# initialize RBM parameters
RBM.initParams()
SGD = sgd.SGD(RBM, epochs=2, alpha=1e-5, minibatch=50)
# run SGD loop
print "Training..."
SGD.run(trainImages)
# view up to 100 learned features post training
W = RBM.W #prp.unwhiten(RBM.W)
vsl.view_patches(W.reshape(imDim, imDim, hiddenSize), min(hiddenSize, 100))
print "Sampling Gibbs chains..."
def testing_load_previous_and_transform(self):
# Note - this doesn't work and i don't know why
# load up our rbms[2]
r = rbm.RBM(3,
2,
visible_unit_type='bin',
model_name=self.name + '_layer_2',
verbose=1,
main_dir='layer_2_test')
r.load_model(
[[3, 2], [22, 3]], 1,
'/home/tyler/2018_Development/Fantasy_Stats_ML/layer_2_test/models/test1_layer_2'
)
test_set = pd.read_csv(
'FFNN_Dev_Testing/L1_training_set_transformed.csv',
sep=',',
header=None)
testing_tools.write_csv(r.transform(test_set),
TESTING_PATH + '/Loaded_L2_transform.csv')
def main():
print 'load data...'
with gzip.open('./dataset/mnist.pkl.gz') as f:
train_set, valid_set, test_set = cPickle.load(f)
train_set_x, train_set_y = shared_data(train_set)
# model params
learning_rate = 0.01
batch_size = 20
epoches = 100
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
x = T.matrix('input')
index = T.lscalar('index')
rng = np.random.RandomState(1234)
print 'initial params...'
RBM = rbm.RBM(x, n_visiable=784, n_hidden=500, numpy_rng=rng)
cost, updates = RBM.cost_updates(lr=learning_rate, k_step=1)
train_model = theano.function(
inputs=[index],
outputs=cost,
updates=updates,
givens={x: train_set_x[index * batch_size:(index + 1) * batch_size]})
print 'training...'
for epoch in xrange(epoches):
t1 = timeit.default_timer()
train_cost = [train_model(i) for i in xrange(n_train_batches)]
print 'epoch = {}, cross-entroy = {}, time = {}'.format(
epoch, np.mean(train_cost),
timeit.default_timer() - t1)
plotting_start = timeit.default_timer()
# Construct image from the weight matrix
if epoch % 5 == 0:
image = Image.fromarray(
utils.tile_raster_images(X=RBM.W.get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=(10, 10),
tile_spacing=(1, 1)))
image.save('./result_image/filters_at_epoch_%i.png' % epoch)
with open('model.pkl', 'w') as f:
cPickle.dump(RBM, f)
print 'save model'
def KL_Div(size, temp, type='1d', epochs=10000, stacked=False, plot=True, k=1):
ratio_hidden = 0.5
if type == '1d':
vis = size
hid = int(vis * ratio_hidden)
if type == '2d':
vis = size**2
if type == '3d':
vis = size**3
hid = int(vis * ratio_hidden)
if stacked == True:
second_hid = int(hid * ratio_hidden)
r = rbm.Stacked_RBM(num_visible=vis,
num_first_hidden=hid,
num_second_hidden=second_hid)
else:
r = rbm.RBM(num_visible=vis, num_hidden=hid)
training_data_sample = np.loadtxt('training_data_temp%s.txt' % temp)
errors = r.train(training_data_sample,
epochs=int(epochs),
learning_rate=0.1,
batch_size=100,
k=k)
if plot == True:
epochs = np.arange(1, epochs + 1)
fig, ax = plt.subplots()
ax.plot(epochs, errors, '.')
plt.title('KL Divergence')
plt.xlabel('Epochs')
plt.ylabel('KL Div')
plt.show()
return epochs, errors
def gen_data_lowT(RBM,
beta=1,
which='marginal',
Nchains=10,
Lchains=100,
Nthermalize=0,
Nstep=1,
N_PT=1,
reshape=True,
update_betas=False,
config_init=[]):
if which == 'joint':
tmp_RBM = copy.deepcopy(RBM)
tmp_RBM.vlayer.fields *= beta
tmp_RBM.weights *= beta
if RBM.hidden in ['Bernoulli', 'Spin']:
tmp_RBM.hlayer.fields *= beta
elif RBM.hidden == 'Gaussian':
tmp_RBM.hlayer.a *= beta
tmp_RBM.hlayer.b *= beta
elif RBM.hidden == 'ReLU+':
tmp_RBM.hlayer.a *= beta
tmp_RBM.hlayer.theta_plus *= beta
elif RBM.hidden == 'ReLU':
tmp_RBM.hlayer.a *= beta
tmp_RBM.hlayer.theta_plus *= beta
tmp_RBM.hlayer.theta_minus *= beta
elif RBM.hidden == 'dReLU':
tmp_RBM.hlayer.a_plus *= beta
tmp_RBM.hlayer.a_minus *= beta
tmp_RBM.hlayer.theta_plus *= beta
tmp_RBM.hlayer.theta_minus *= beta
elif which == 'marginal':
if type(beta) == int:
tmp_RBM = rbm.RBM(n_v=RBM.n_v,
n_h=beta * RBM.n_h,
visible=RBM.visible,
hidden=RBM.hidden,
n_cv=RBM.n_cv,
n_ch=RBM.n_ch)
tmp_RBM.vlayer.fields = beta * RBM.vlayer.fields
tmp_RBM.vlayer.fields0 = RBM.vlayer.fields0
tmp_RBM.weights = np.repeat(RBM.weights, beta, axis=0)
if RBM.hidden in ['Bernoulli', 'Spin']:
tmp_RBM.hlayer.fields = np.repeat(RBM.hlayer.fields,
beta,
axis=0)
tmp_RBM.hlayer.fields0 = np.repeat(RBM.hlayer.fields0,
beta,
axis=0)
elif RBM.hidden == 'Gaussian':
tmp_RBM.hlayer.a = np.repeat(RBM.hlayer.a, beta, axis=0)
tmp_RBM.hlayer.a0 = np.repeat(RBM.hlayer.a0, beta, axis=0)
tmp_RBM.hlayer.b = np.repeat(RBM.hlayer.b, beta, axis=0)
tmp_RBM.hlayer.b0 = np.repeat(RBM.hlayer.b0, beta, axis=0)
elif RBM.hidden == 'ReLU+':
tmp_RBM.hlayer.a = np.repeat(RBM.hlayer.a, beta, axis=0)
tmp_RBM.hlayer.a0 = np.repeat(RBM.hlayer.a0, beta, axis=0)
tmp_RBM.hlayer.theta_plus = np.repeat(RBM.hlayer.theta_plus,
beta,
axis=0)
tmp_RBM.hlayer.theta_plus0 = np.repeat(RBM.hlayer.theta_plus0,
beta,
axis=0)
elif RBM.hidden == 'ReLU':
tmp_RBM.hlayer.a = np.repeat(RBM.hlayer.a, beta, axis=0)
tmp_RBM.hlayer.a0 = np.repeat(RBM.hlayer.a0, beta, axis=0)
tmp_RBM.hlayer.theta_plus = np.repeat(RBM.hlayer.theta_plus,
beta,
axis=0)
tmp_RBM.hlayer.theta_plus0 = np.repeat(RBM.hlayer.theta_plus0,
beta,
axis=0)
tmp_RBM.hlayer.theta_minus = np.repeat(RBM.hlayer.theta_minus,
beta,
axis=0)
tmp_RBM.hlayer.theta_minus0 = np.repeat(
RBM.hlayer.theta_minus0, beta, axis=0)
elif RBM.hidden == 'dReLU':
tmp_RBM.hlayer.a_plus = np.repeat(RBM.hlayer.a_plus,
beta,
axis=0)
tmp_RBM.hlayer.a_plus0 = np.repeat(RBM.hlayer.a_plus0,
beta,
axis=0)
tmp_RBM.hlayer.a_minus = np.repeat(RBM.hlayer.a_minus,
beta,
axis=0)
tmp_RBM.hlayer.a_minus0 = np.repeat(RBM.hlayer.a_minus0,
beta,
axis=0)
tmp_RBM.hlayer.theta_plus = np.repeat(RBM.hlayer.theta_plus,
beta,
axis=0)
tmp_RBM.hlayer.theta_plus0 = np.repeat(RBM.hlayer.theta_plus0,
beta,
axis=0)
tmp_RBM.hlayer.theta_minus = np.repeat(RBM.hlayer.theta_minus,
beta,
axis=0)
tmp_RBM.hlayer.theta_minus0 = np.repeat(
RBM.hlayer.theta_minus0, beta, axis=0)
return tmp_RBM.gen_data(Nchains=Nchains,
Lchains=Lchains,
Nthermalize=Nthermalize,
Nstep=Nstep,
N_PT=N_PT,
reshape=reshape,
update_betas=update_betas,
config_init=config_init)
def init_weight(shape):
return tf.Variable(tf.random_normal(shape, mean=0.0, stddev=0.01))
def init_bias(dim):
return tf.Variable(tf.zeros([dim]))
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
X = tf.placeholder("float", [None, 784])
Y = tf.placeholder("float", [None, 10])
rbm_layer = rbm.RBM("mnist", 784, 500)
for i in range(1):
print "RBM CD: ", i
rbm_layer.cd1(trX)
rbm_w, rbm_vb, rbm_hb = rbm_layer.cd1(trX)
wo = init_weight([500, 10])
bo = init_bias(10)
py_x = build_model(X, rbm_w, rbm_hb, wo, bo)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(py_x, Y))
train_op = tf.train.GradientDescentOptimizer(0.05).minimize(cost)
predict_op = tf.argmax(py_x, 1)
labels_train = ml_common.loadMNISTLabels(r"common/train-labels-idx1-ubyte")
images_train = ml_common.loadMNISTImages(r"common/train-images-idx3-ubyte")
imShape = (images_train.shape[1], images_train.shape[2])
images_train = images_train.reshape(
images_train.shape[0], images_train.shape[1] * images_train.shape[2])
# "Real" data.
labels_real = ml_common.loadMNISTLabels(r"common/t10k-labels-idx1-ubyte")
images_real = ml_common.loadMNISTImages(r"common/t10k-images-idx3-ubyte")
images_real = images_real.reshape(images_real.shape[0],
images_real.shape[1] * images_real.shape[2])
# Train the RBM.
if not doMultipleLayers:
if doReadRBMFromFile:
rbm = rbm.RBM(0, 0).read(os.path.join(tmpdir, r"rbm.pickle"))
np.random.set_state(rbm.rngState)
else:
rbm = rbm.RBM(images_train.shape[1],
150,
actType=actType,
batchSize=500)
if doTrainRBM:
rbm.learn(images_train, maxIter=20000, rate=0.01, wDecay=0.01)
rbm.write(os.path.join(tmpdir, "rbmb.pickle"))
rbm.viewWeights(imShape, outfile=os.path.join(tmpdir, r"weights.png"))
else:
if doReadRBMFromFile:
rbms = rbm_stacked.MultRBM([]).read(
import rbm
import pandas as pd
import testing_tools as tt
# import our data (numerical values only)
TEST_SET_PATH = "FormattedFantasyData/2018_data.csv"
test_set = pd.read_csv(TEST_SET_PATH, sep=',', header=None)
test_set = test_set.values
r = rbm.RBM(22,
2,
visible_unit_type='gauss',
model_name="test_model",
verbose=1,
main_dir='sametest')
r.fit(test_set)
#tt.write_csv(r.get_model_parameters() , 'model_made_data_')
#tt.write_csv(r.transform(df) , 'model_trained_data_transformed_1.csv')
print(r.get_model_parameters())
def Generate_and_Test(size,
K,
length,
number_train,
number_gen,
min_temp,
max_temp,
number_temps,
type='1d',
plot=True,
gen_training=True,
epochs=100,
stacked=False,
k=1):
if min_temp == 0.0:
min_temp = 0.00001
temperatures = np.linspace(min_temp, max_temp, number_temps)
training_stats = np.zeros((5, number_temps))
if isinstance(epochs, (list, tuple, np.ndarray)) == False:
epochs_for_temps = np.zeros(number_temps)
epochs_for_temps.fill(epochs)
else:
epochs_for_temps = epochs
training_data = []
if gen_training == True:
for i in range(number_temps):
#print(Training_Data(size,temperatures[i],K,length,number_train,type=type))
training_data.append(
Training_Data(size,
temperatures[i],
K,
length,
number_train,
type=type))
np.savetxt('training_data_temp%s.txt' % temperatures[i],
training_data[i])
#np.savetxt('training_data.txt',training_data)
ratio_hidden = 0.5
autocorrelation_guess_rbm = 100
#print(training_data)
gen_data = []
for i in range(number_temps):
if type == '1d':
vis = size
hid = int(vis * ratio_hidden)
if type == '2d':
vis = size**2
if type == '3d':
vis = size**3
hid = int(vis * ratio_hidden)
if stacked == True:
second_hid = int(hid * ratio_hidden)
r = rbm.Stacked_RBM(num_visible=vis,
num_first_hidden=hid,
num_second_hidden=second_hid)
else:
r = rbm.RBM(num_visible=vis, num_hidden=hid)
#print(training_data[i])
if gen_training == True:
r.train(training_data[i],
epochs=int(epochs_for_temps[i]),
learning_rate=0.1,
batch_size=100,
k=k)
gen_data.append(
r.daydream(number_gen * autocorrelation_guess_rbm,
training_data[i][0]))
else:
training_data_sample = np.loadtxt('training_data_temp%s.txt' %
temperatures[i])
r.train(training_data_sample,
epochs=int(epochs_for_temps[i]),
learning_rate=0.1,
batch_size=100,
k=k)
gen_data.append(
r.daydream(number_gen * autocorrelation_guess_rbm,
training_data_sample[0]))
training_stats[0][i] = temperatures[i]
training_stats[1][i], training_stats[2][i], training_stats[3][
i], training_stats[4][i] = Test_Generated(training_data_sample,
size,
temperatures[i],
K,
type=type)
#np.savetxt('generated_data_temp%s.txt'%temperatures[i],gen_data[i])
test_data = np.zeros((5, number_temps))
for i in range(number_temps):
M, E, susept, specific_heat = Test_Generated(
gen_data[i],
size,
temperatures[i],
K,
type=type,
autocorrelation=autocorrelation_guess_rbm)
test_data[0][i] = temperatures[i]
test_data[1][i] = M
test_data[2][i] = E
test_data[3][i] = susept
test_data[4][i] = specific_heat
if gen_training == True:
training_stats[1][i], training_stats[2][i], training_stats[3][
i], training_stats[4][i] = Test_Generated(training_data[i],
size,
temperatures[i],
K,
type=type)
# Want to compare stats of MC and RBM data
differences = np.zeros((5, number_temps))
differences[0] = np.abs(training_stats[1] - test_data[1])
differences[1] = np.abs(training_stats[2] - test_data[2])
differences[2] = np.abs(training_stats[3] - test_data[3])
differences[3] = np.abs(training_stats[4] - test_data[4])
if plot == True:
def analytic_energy(temp):
return -K * np.tanh(temp**(-1) * K)
def analytic_Cv(temp):
return spc.k * (temp**(-1) * K)**2 * (np.cosh(
temp**(-1) * K))**(-2)
fig, ax = plt.subplots()
plt.plot(temperatures, differences[0], '.', label='magnetization')
plt.plot(temperatures, differences[1], '.', label='energy')
plt.plot(temperatures, differences[2], '.', label='susceptibility')
plt.plot(temperatures, differences[3], '.', label='specific heat')
plt.legend()
plt.ylim(0, 1)
plt.xlabel('Fundamental Temperature')
plt.ylabel('Difference between MC and RBM Stats')
fig, ax = plt.subplots()
plt.plot(training_stats[0], training_stats[2], '.', label='MC')
plt.plot(test_data[0], test_data[2], '.', label='rbm data')
if type == '1d':
plt.plot(test_data[0],
analytic_energy(test_data[0]),
'.',
label='Exact')
plt.legend()
plt.xlabel('Fundamental Temperature')
plt.ylabel('Average Energy per cell after Convergence')
fig, ax = plt.subplots()
plt.plot(training_stats[0], training_stats[1], '.', label='MC')
plt.plot(test_data[0], test_data[1], '.', label='rbm data')
plt.legend()
plt.xlabel('Fundamental Temperature')
plt.ylabel('Average Magnetisation per cell after Convergence')
fig, ax = plt.subplots()
plt.plot(training_stats[0], training_stats[3], '.', label='MC')
plt.plot(test_data[0], test_data[3], '.', label='rbm data')
plt.legend()
plt.xlabel('Fundamental Temperature')
plt.ylabel('Suseptibility')
fig, ax = plt.subplots()
plt.plot(training_stats[0], training_stats[4], '.', label='MC')
plt.plot(test_data[0], test_data[4], '.', label='rbm data')
plt.legend()
plt.xlabel('Fundamental Temperature')
plt.ylabel('Specific Heat')
plt.show()
return differences
def Autocorrelations_in_generated(size,
K,
max_binsize,
min_samples,
temp,
type='1d',
plot=True,
epochs=100,
stacked=False,
k=1):
ratio_hidden = 0.5
gen_data = []
if type == '1d':
vis = size
hid = int(vis * ratio_hidden)
if type == '2d':
vis = size**2
if type == '3d':
vis = size**3
hid = int(vis * ratio_hidden)
if stacked == True:
second_hid = int(hid * ratio_hidden)
r = rbm.Stacked_RBM(num_visible=vis,
num_first_hidden=hid,
num_second_hidden=second_hid)
else:
r = rbm.RBM(num_visible=vis, num_hidden=hid)
training_data_sample = np.loadtxt('training_data_temp%s.txt' % temp)
r.train(training_data_sample,
epochs=int(epochs),
learning_rate=0.1,
batch_size=100,
k=k)
number_gen = max_binsize * min_samples
gen_data = r.daydream(number_gen, training_data_sample[0])
binsizes = np.arange(1, max_binsize + 1)
magnetisations = []
standard_dev_magnetisations = []
for binsize_index in range(max_binsize):
samples = int(number_gen / binsizes[binsize_index])
bin_magnetizations = np.zeros((samples))
for sample in range(samples):
for i in range(binsizes[binsize_index]):
Ising_lattice = lattice.Initialise_Random_State(size,
temp,
K,
type=type)
Ising_lattice.set_spins(
gen_data[sample * binsizes[binsize_index] + i])
bin_magnetizations[sample] += np.abs(
Ising_lattice.magnetisation())
bin_magnetizations[
sample] = bin_magnetizations[sample] / binsizes[binsize_index]
mean = np.mean(bin_magnetizations)
magnetisations.append(mean)
difference_from_mean = np.sum(np.square(bin_magnetizations - mean))
st_dev = (difference_from_mean / (samples * (samples - 1)))**0.5
standard_dev_magnetisations.append(st_dev)
fig, ax = plt.subplots()
ax.errorbar(binsizes,
magnetisations,
fmt='-o',
yerr=standard_dev_magnetisations)
plt.title('Determining Autocorrelations')
plt.xlabel('Binsizes')
plt.ylabel('Magnetization')
plt.show()
import rbm
import tensorflow as tf
import numpy as np
input_matrix, labels = p.main()
print "Input matrix shape = ", input_matrix.shape[0]
print "labels shape = ", labels.shape[0]
print labels[0, 0:10]
#for row in input_matrix:
visible = input_matrix[0]
hidden = labels[0]
vis = tf.Variable(visible)
r = rbm.RBM("chr0.0", visible.shape[0], hidden.shape[0])
with tf.Session() as session:
# Run the model
#session.run(r)
session.run(r.propup(vis))
# Run just the variable y and print
#print(session.run(y))
#sess.Run(rbm
#x = RBM("test",
#rbm = tf.Variable(x+5, name = 'y')
#sess = tf.Session()
#sess.Run(
W = 100 * (opts.cols + 1) + 4
H = 100 * opts.rows + 4
win = glumpy.Window(W, H)
loaded = False
updates = -1
batches = 0.
recent = collections.deque(maxlen=20)
errors = [collections.deque(maxlen=20) for _ in range(10)]
testset = [None] * 10
trainset = dict((i, []) for i in range(10))
loader = idx_reader.iterimages(opts.labels, opts.images, False)
rbm = opts.model and pickle.load(open(opts.model, 'rb')) or rbm.RBM(
28 * 28, opts.rows * opts.cols, opts.binary)
trainer = rbm.Trainer(
rbm,
momentum=opts.momentum,
target_sparsity=opts.sparsity,
)
def get_pixels():
global loaded
if not loaded and numpy.all([len(trainset[t]) > 10
for t in range(10)]):
loaded = True
if loaded and rng.random() < 0.99:
t = rng.randint(10)
TEST_SET_PATH = "FormattedFantasyData/2018_data.csv"
NUM_DATAPOINTS = 22
# import our training dataset into a dataframe, then immediately recast it to numpy array
# this dataset contains fantasy stats for the past decate (excluding this current season)
# and is numerical only
# the data has 22 points of data for each player
training_set = pd.read_csv(TRAINING_SET_PATH, sep=',', header=None)
training_set = training_set.values
# get our test set
test_set = pd.read_csv(TEST_SET_PATH, sep=',', header=None)
test_set = test_set.values
# initialize RBM
r = rbm.RBM(NUM_DATAPOINTS,
3,
visible_unit_type='gauss',
model_name="fantasy_position",
verbose=1,
main_dir='fantasy_test')
# fit for training set
r.fit(training_set)
# see what this puppy thinks of the test set
testing_tools.write_csv(r.transform(test_set),
'RBMTestingResults/2018_data_transform_first.csv')
print("finished")
