def recommend(self, u, q):
self.average_weights = self._load_matrix(WEIGHTS_FILE)
self.average_bv = self._load_matrix(VISIBLE_BIAS_FILE)
self.average_bh = self._load_matrix(HIDDEN_BIAS_FILE)
#x is the input (the user we are predicting for), w is the weights of just the items rated by user x and bv is
# just the visible biases of the corresponding to the items rated by the user
x, w, bv = self._ratings_to_softmax_units(self.dataset.training_X[:,u], q)
#bh is the same size for every user because they correspond to the hidden units
bh = self.average_bh
#Returns the new index for the query q. The index changes because we reomved all unrated items.
#The units for the query are changed to 5 -1s as a placeholder
x, new_q = self._new_index(x)
results = []
#For each rating 1-5 we sample the RBM with the corresponding input.
for r in range(RATING_MAX):
x = self._set_rating(x, new_q, (r+1))
rbm = RBM(hidden_layer_n=HIDDEN_LAYER_N,iterations=ITERATIONS,dataset=x, training=False)
sample = rbm.run_prediction(x, w, bh, bv)
results.append(sample)
#Get the expected output from each of the probablitlies of each input
probs = self._get_probabilities(results, new_q)
prediction = self._expected_value(self.softmax(probs))
print("Prediction: user " + str(u) + " will give movie: " + str(q) + " rating: " + str(prediction))
def train(self):
#wb is a dictionary that stores the average Weight matrices and Bias matrices. The keys are where the
#files are stored
self.wb = {}
"""
For each user an RBM will be created.
"""
for i in range(self.dataset.training_X.shape[1]):
user = self.dataset.training_X[:,i]
rbm = RBM(hidden_layer_n=HIDDEN_LAYER_N,iterations=ITERATIONS,dataset=user)
rbm.run()
#After the an RBM is run the weights and biases are re-added to complete set.
self.all_weights.append(rbm.full_weights)
self.all_bv.append(rbm.full_bv)
self.all_bh.append(rbm.full_bh)
print("RBM number: " + str(i))
#Average all the weights and all the biases from all the RBM's (With each RBM corresponding to a user)
self.wb[WEIGHTS_FILE] = self.average_matrices(self.all_weights)
self.wb[VISIBLE_BIAS_FILE] = self.average_matrices(self.all_bv)
self.wb[HIDDEN_BIAS_FILE] = self.average_matrices(self.all_bh)
#Training can take a long time so we can save the weights and biases
self.save_matrix(WEIGHTS_FILE)
self.save_matrix(VISIBLE_BIAS_FILE)
self.save_matrix(HIDDEN_BIAS_FILE)
def fit(self, trainset):
AlgoBase.fit(self, trainset)
numUsers = trainset.n_users
numItems = trainset.n_items
trainingMatrix = np.zeros([numUsers, numItems, 10], dtype = np.float32)
for (uid, iid, rating) in trainset.all_ratings():
adjustedRating = int(float(rating)*2.0) - 1
trainingMatrix[int(uid), int(iid), adjustedRating] = 1
trainingMatrix = np.reshape(trainingMatrix,[trainingMatrix.shape[0], - 1])
rbm = RBM(trainingMatrix.shape[1], hiddenDimensions = self.hiddenDim, learningRate = self.learningRate, batchSize = self.batchSize)
rbm.Train(trainingMatrix)
self.predictedRatings = np.zeros([numUsers, numItems], dtype = np.float32)
for uiid in range(trainset.n_users):
if(uiid % 50 == 0):
print("Procissing user ", uiid)
recs = rbm.GetRecommendations([trainingMatrix[uiid]])
recs = np.reshape(recs, [numItems, 10])
for itemID, rec in enumerate(recs):
normalized = self.softmax(rec)
rating = np.average(np.arange(10), weights = normalized)
self.predictedRatings[uiid,itemID] = (rating + 1)* 0.5
return self
def __init__(self,
input=None,
label=None,
n_ins=2,
hidden_layer_sizes=[3, 3],
n_outs=2,
rng=None):
self.x = input
self.y = label
self.sigmoid_layers = []
self.rbm_layers = []
self.n_layers = len(hidden_layer_sizes) # = len(self.rbm_layers)
if rng is None:
rng = numpy.random.RandomState(1234)
assert self.n_layers > 0
# construct multi-layer
for i in xrange(self.n_layers):
# layer_size
if i == 0:
input_size = n_ins
else:
input_size = hidden_layer_sizes[i - 1]
# layer_input
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].sample_h_given_v()
# construct sigmoid_layer
sigmoid_layer = HiddenLayer(input=layer_input,
n_in=input_size,
n_out=hidden_layer_sizes[i],
rng=rng,
activation=sigmoid)
self.sigmoid_layers.append(sigmoid_layer)
# construct rbm_layer
rbm_layer = RBM(
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layer_sizes[i],
W=sigmoid_layer.W, # W, b are shared
hbias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
# layer for output using Logistic Regression
self.log_layer = LogisticRegression(
input=self.sigmoid_layers[-1].sample_h_given_v(),
label=self.y,
n_in=hidden_layer_sizes[-1],
n_out=n_outs)
# finetune cost: the negative log likelihood of the logistic regression layer
self.finetune_cost = self.log_layer.negative_log_likelihood()
def fit(self, trainset):
AlgoBase.fit(self, trainset)
numUsers = trainset.n_users
numItems = trainset.n_items
trainingMatrix = np.zeros([numUsers, numItems, 10], dtype=np.float32)
for (uid, iid, rating) in trainset.all_ratings():
adjustedRating = int(float(rating)*2.0) - 1
trainingMatrix[int(uid), int(iid), adjustedRating] = 1
# Flatten to a 2D array, with nodes for each possible rating type on each possible item, for every user.
trainingMatrix = np.reshape(trainingMatrix, [trainingMatrix.shape[0], -1])
# Create an RBM with (num items * rating values) visible nodes
rbm = RBM(trainingMatrix.shape[1], hiddenDimensions=self.hiddenDim, learningRate=self.learningRate, batchSize=self.batchSize, epochs=self.epochs)
rbm.Train(trainingMatrix)
self.predictedRatings = np.zeros([numUsers, numItems], dtype=np.float32)
for uiid in range(trainset.n_users):
if (uiid % 50 == 0):
print("Processing user ", uiid)
recs = rbm.GetRecommendations([trainingMatrix[uiid]])
recs = np.reshape(recs, [numItems, 10])
for itemID, rec in enumerate(recs):
# The obvious thing would be to just take the rating with the highest score:
#rating = rec.argmax()
# ... but this just leads to a huge multi-way tie for 5-star predictions.
# The paper suggests performing normalization over K values to get probabilities
# and take the expectation as your prediction, so we'll do that instead:
normalized = self.softmax(rec)
rating = np.average(np.arange(10), weights=normalized)
self.predictedRatings[uiid, itemID] = (rating + 1) * 0.5
return self
def train_DBN(self, x):
for index, layer in enumerate(self.layers):
if index == 0:
vn = self.input_size
else:
vn = self.layers[index - 1]
hn = self.layers[index]
rbm = RBM(vn,
hn,
epochs=100,
mode='bernoulli',
lr=0.0005,
k=10,
batch_size=128,
gpu=True,
optimizer='adam',
early_stopping_patience=10)
x_dash = self.generate_input_for_layer(index, x)
rbm.train(x_dash)
self.layer_parameters[index]['W'] = rbm.W.cpu()
self.layer_parameters[index]['hb'] = rbm.hb.cpu()
self.layer_parameters[index]['vb'] = rbm.vb.cpu()
print("Finished Training Layer:", index, "to", index + 1)
if self.savefile is not None:
torch.save(self.layer_parameters, self.savefile)
def pretrain_model(self, x):
with tf.variable_scope('pre-train_dbn'):
# Make an rbm between each layer of the dbn so that
# we can train each layer greedily
with tf.variable_scope('make_rbms'):
rbm_layers = [x]
rbms = []
for i in range(len(self.dbn_sizes) - 1):
rbms.append(RBM(self.W[i], self.B[i], self.B[i + 1]))
visible_layer = tf.sigmoid(
tf.matmul(rbm_layers[-1], self.W[i]) + self.B[i + 1])
rbm_layers.append(sample(visible_layer))
# Create a list of optimizers and other subgraphs, one for each rbm
# that we will train
with tf.variable_scope('pre-train_ops'):
costs = []
optimizers = []
for i in range(len(rbms)):
cost, _ = rbms[i].free_energy_cost(rbm_layers[i], 1)
optimizer = tf.train.AdamOptimizer().minimize(cost)
costs.append(cost)
optimizers.append(optimizer)
return costs, optimizers
def __init__(self,
visible_units=256,
hidden_units=[64, 100],
k=2,
learning_rate=1e-5,
learning_rate_decay=False,
xavier_init=False,
increase_to_cd_k=False,
use_gpu=False):
super(DBN, self).__init__()
self.n_layers = len(hidden_units)
self.rbm_layers = []
self.rbm_nodes = []
# Creating different RBM layers
for i in range(self.n_layers):
input_size = 0
if i == 0:
input_size = visible_units
else:
input_size = hidden_units[i - 1]
rbm = RBM(visible_units=input_size,
hidden_units=hidden_units[i],
k=k,
learning_rate=learning_rate,
learning_rate_decay=learning_rate_decay,
xavier_init=xavier_init,
increase_to_cd_k=increase_to_cd_k,
use_gpu=use_gpu)
self.rbm_layers.append(rbm)
# rbm_layers = [RBM(rbn_nodes[i-1] , rbm_nodes[i],use_gpu=use_cuda) for i in range(1,len(rbm_nodes))]
self.W_rec = [
nn.Parameter(self.rbm_layers[i].W.data.clone())
for i in range(self.n_layers - 1)
]
self.W_gen = [
nn.Parameter(self.rbm_layers[i].W.data)
for i in range(self.n_layers - 1)
]
self.bias_rec = [
nn.Parameter(self.rbm_layers[i].h_bias.data.clone())
for i in range(self.n_layers - 1)
]
self.bias_gen = [
nn.Parameter(self.rbm_layers[i].v_bias.data)
for i in range(self.n_layers - 1)
]
self.W_mem = nn.Parameter(self.rbm_layers[-1].W.data)
self.v_bias_mem = nn.Parameter(self.rbm_layers[-1].v_bias.data)
self.h_bias_mem = nn.Parameter(self.rbm_layers[-1].h_bias.data)
for i in range(self.n_layers - 1):
self.register_parameter('W_rec%i' % i, self.W_rec[i])
self.register_parameter('W_gen%i' % i, self.W_gen[i])
self.register_parameter('bias_rec%i' % i, self.bias_rec[i])
self.register_parameter('bias_gen%i' % i, self.bias_gen[i])
def pretrain_autoencoder(net,
x,
x_val,
rbm_lr=0.001,
rbm_use_gauss_visible=False,
rbm_use_gauss_hidden=True,
rbm_mom=0.5,
rbm_weight_decay=0.0000,
rbm_lr_decay=0.0,
rbm_batch_size=100,
rbm_epochs=100,
rbm_patience=-1,
verbose=1):
final_arch = net.arch[1:math.ceil(len(net.arch) /
2.0)] # without input layer
n_dense_layers = len(final_arch)
rbm_list = []
#loop for training the RBMs
for i in range(n_dense_layers):
print("\nFine tuning layer number " + str(i))
if (i == 0):
x_new = x
x_val_new = x_val
else:
x_new = rbm_list[-1].get_h(x_new)
x_val_new = rbm_list[-1].get_h(x_val_new)
rbm = RBM(x_new.shape[1],
final_arch[i],
use_gaussian_visible_sampling=rbm_use_gauss_visible,
use_gaussian_hidden_sampling=rbm_use_gauss_hidden,
use_sample_vis_for_learning=False)
rbm.set_lr(
rbm_lr,
rbm_lr_decay,
momentum=rbm_mom,
weight_decay=rbm_weight_decay,
)
rbm.fit(x_new,
x_val_new,
batch_size=rbm_batch_size,
epochs=rbm_epochs,
patience=rbm_patience)
rbm_list.append(rbm)
rbm_iterator = 0
rbm_iterator_direction = 1
#loop to copy the weights from rbm to NN
for n_layer in range(len(net.layers)):
if (net.layers[n_layer].ID == "Dense"):
copy_dense_weights_from_rbm(rbm_list[rbm_iterator],
net.layers[n_layer],
rbm_iterator_direction)
if (rbm_iterator == len(rbm_list) - 1
and rbm_iterator_direction == 1):
rbm_iterator_direction = -1
else:
rbm_iterator += rbm_iterator_direction
print("Pre training finished!")
return rbm_list
def __init__(self, layer_sizes):
"""On va traiter le DBN comme si c'etait une liste des RBM
Paramètres: nb_couches doit être int > 0
"""
self.nb_couches = len(layer_sizes) - 1
# Un DBN est un empilement de RBM
self.model = []
for i in range(self.nb_couches):
self.model.append(RBM(layer_sizes[i], layer_sizes[i + 1]))
def __init__(self,
numpy_rng,
theano_rng=None,
n_ins=784,
hidden_layers_sizes=[500, 500],
n_outs=10):
self.sigmoid_layers = []
self.rbm_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = theano_rng = RandomStreams(numpy_rng.randint(2**30))
self.x = T.matrix('x')
self.y = T.ivector('y')
for i in range(self.n_layers):
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
self.sigmoid_layers.append(sigmoid_layer)
self.params.extend(sigmoid_layer.params)
rbm_layer = RBM(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
self.logLayer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.params.extend(self.logLayer.params)
self.finetune_cost = self.logLayer.negative_log_likehood(self.y)
self.errors = self.logLayer.errors(self.y)
def train_model(self, x):
with tf.variable_scope('train_rnn_dbn'):
with tf.variable_scope('propagate_states'):
states = self.__unroll_rnn(x)
state0 = self.rnn_s0
if self.num_rnn_cells > 1:
states = states[-1]
state0 = state0[-1]
u_t = tf.reshape(states.c, [-1, self.s_size])
q_t = tf.reshape(states.h, [-1, self.s_size])
u_tm1 = tf.concat([state0.c, u_t], 0)[:-1, :]
q_tm1 = tf.concat([state0.h, q_t], 0)[:-1, :]
# Make an rbm between each layer of the dbn so that
# we can train each layer greedily
with tf.variable_scope('make_rbms'):
rbm_layers = [x]
rbms = []
for i in range(len(self.dbn_sizes) - 1):
bv = tf.matmul(u_tm1, self.Wu[i]) + tf.matmul(
q_tm1, self.Wq[i]) + self.B[i]
bh = tf.matmul(u_tm1, self.Wu[i + 1]) + tf.matmul(
q_tm1, self.Wq[i + 1]) + self.B[i + 1]
rbms.append(RBM(self.W[i], bv, bh))
visible_layer = tf.sigmoid(
tf.matmul(rbm_layers[-1], self.W[i]) + self.B[i + 1])
rbm_layers.append(sample(visible_layer))
# Create a list of optimizers and other subgraphs, one for each rbm
# that we will train
with tf.variable_scope('train_ops'):
costs = []
optimizers = []
loglikelihoods = []
summaries = []
for i in range(len(rbms)):
cost, loglikelihood = rbms[i].free_energy_cost(
rbm_layers[i], 15)
cost_summary = tf.summary.scalar('free_energy', cost)
ll_summary = tf.summary.scalar('log_likelihood', loglikelihood)
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=0.001)
gradients = optimizer.compute_gradients(cost)
gradients = [(tf.clip_by_value(grad, -10.0, 10.0), var)
for grad, var in gradients if grad is not None]
optimizer = optimizer.apply_gradients(gradients)
costs.append(cost)
optimizers.append(optimizer)
loglikelihoods.append(loglikelihood)
summaries.append([cost_summary, ll_summary])
return costs, optimizers, loglikelihoods, summaries
def main():
learningRate = float(sys.argv[1]) if len(sys.argv) >= 2 else 0.0001
maxIterations = int(sys.argv[2]) if len(sys.argv) >= 3 else 300
# load data
data = scipy.io.loadmat('data/usps_resampled.mat')
train_patterns = data['train_patterns']
train_labels = data['train_labels']
test_patterns = data['test_patterns']
test_labels = data['test_labels']
# initialize and train RBM
rbm = RBM(192, train_patterns, learningRate=learningRate, verbose=True)
iterationsCompleted = rbm.train(convThreshold=0.03,
maxIterations=maxIterations)
print 'Autoencoding. . . '
hidden_patterns = rbm.translate(train_patterns)
ae_patterns = rbm.invert(hidden_patterns)
print 'Finished.'
while True:
while True:
try:
sampleImage = raw_input("Pick a sample image from [0-" +
str(train_patterns.shape[1] - 1) +
"] (q to quit): ")
if sampleImage == 'q':
y = raw_input("Save this classifier (y)? ")
fn = 'data/RBM_' + str(
(learningRate, 192, iterationsCompleted))
if y in ['y', '']:
f = open(fn, 'w')
pickle.dump(TrainedRBM(rbm.W, rbm.b_h, rbm.b_v), f)
print "RBM saved as " + fn
sys.exit(0)
sampleImage = int(sampleImage)
if sampleImage not in range(train_patterns.shape[1]):
raise ValueError
except ValueError:
continue
except EOFError:
sys.exit(0)
except KeyboardInterrupt:
sys.exit(0)
break
# show example image
plt.matshow(train_patterns[:, sampleImage].reshape(16, 16))
plt.matshow(ae_patterns[:, sampleImage].reshape(16, 16))
plt.show()
def fit(self, X, transform=False):
self.layer = []
X_input = X
input_size = X_input.shape[1]
self.input_shape = input_size
for i in range(self.n_layers):
output_size = self.params[i]
print(input_size)
self.layer.append(RBM(input_size, output_size))
input_size = output_size
self.layer[i].fit(X_input, self.n_iter[i])
X_input = self.layer[i].predict(X_input)
if transform:
return self.predict(X)
def train(self, trainingData):
rbmList = [] # list RBM's weights
tempData = trainingData
# start RBM's training and get the respective weights
for n in range(self.nEncoderLayers):
if (n == 0 or n == (self.nEncoderLayers - 1)):
rbm = RBM(tempData, self.sEncoderLayers[n], rbmType='GBRBM')
else:
rbm = RBM(tempData, self.sEncoderLayers[n], rbmType='BBRBM')
print('Start %d RBM training' % (n + 1))
rbm.train(batchSize=100)
[weights, visBias, hidBias] = rbm.getWeights()
rbmList.append(RBM_Weights(weights, visBias, hidBias))
data = tf.convert_to_tensor(tempData,
dtype=tf.float32,
name='data')
probHid = tf.sigmoid(tf.matmul(data, weights) + hidBias)
hid = tf.cast(tf.greater(
probHid,
tf.random_uniform(tf.shape(probHid),
minval=0,
maxval=1,
dtype=tf.float32)),
dtype=tf.float32)
with tf.Session() as sess:
if ((self.nEncoderLayers - 1) == n):
tempData = sess.run(probHid)
else:
tempData = sess.run(hid)
# start the fine tuning process
return self.fineTuning(rbmList, trainingData)
def __init__(self,
hidden_units,
visible_units=256,
output_units=1,
k=2,
learning_rate=1e-5,
learning_rate_decay=False,
increase_to_cd_k=False,
device='cpu'):
super(DBN, self).__init__()
self.n_layers = len(hidden_units)
self.rbm_layers = []
self.rbm_nodes = []
self.device = device
self.is_pretrained = False
self.is_finetune = False
# Creating different RBM layers
for i in range(self.n_layers):
if i == 0:
input_size = visible_units
else:
input_size = hidden_units[i - 1]
rbm = RBM(visible_units=input_size,
hidden_units=hidden_units[i],
k=k,
learning_rate=learning_rate,
learning_rate_decay=learning_rate_decay,
increase_to_cd_k=increase_to_cd_k,
device=device)
self.rbm_layers.append(rbm)
self.W_rec = [self.rbm_layers[i].weight for i in range(self.n_layers)]
self.bias_rec = [
self.rbm_layers[i].h_bias for i in range(self.n_layers)
]
for i in range(self.n_layers):
self.register_parameter('W_rec%i' % i, self.W_rec[i])
self.register_parameter('bias_rec%i' % i, self.bias_rec[i])
self.bpnn = torch.nn.Linear(hidden_units[-1],
output_units).to(self.device)
def test_RBM(train_X, train_Y, test_X, test_Y):
# Create and train RBM
rbm = RBM(28 * 28, 500)
rbm.fit(train_X)
rbm.get_filters()
train_X = np.array([rbm.inference(x) for x in train_X])
test_X = np.array([rbm.inference(x) for x in test_X])
logreg = LogisticRegression(max_iter=10)
logreg.fit(train_X, train_Y)
predict_Y = logreg.predict(train_X)
print("Accuracy on training data")
print(accuracy_score(train_Y, predict_Y))
predict_Y = logreg.predict(test_X)
print("Accuracy on test data")
print(accuracy_score(test_Y, predict_Y))
def main():
learningRate = float(sys.argv[1]) if len(sys.argv) >= 2 else 0.0001
maxIterations = int(sys.argv[2]) if len(sys.argv) >= 3 else 300
# load data
data = scipy.io.loadmat('data/usps_resampled.mat')
train_patterns = data['train_patterns']
train_labels = data['train_labels']
test_patterns = data['test_patterns']
test_labels = data['test_labels']
# initialize and train RBM
rbm = RBM(192, train_patterns, learningRate=learningRate, verbose=True)
rbm.train(convThreshold=0.1, maxIterations=maxIterations)
print 'Autoencoding. . . '
hidden_patterns = rbm.translate(train_patterns)
ae_patterns = rbm.invert(hidden_patterns)
print 'Finished.'
W = rbm.W.get_value()
while True:
while True:
try:
detectorNum = raw_input("Pick a detector image from [0-" +
str(W.shape[0] - 1) +
"] (q to quit): ")
if detectorNum == 'q':
sys.exit(0)
detectorNum = int(detectorNum)
if detectorNum not in range(W.shape[0]):
raise ValueError
except ValueError:
continue
except EOFError:
sys.exit(0)
except KeyboardInterrupt:
sys.exit(0)
break
# show example image
plt.matshow(W[detectorNum, :].reshape(16, 16))
plt.show()
def fit(self, trainset):
AlgoBase.fit(self, trainset)
numUsers = trainset.n_users
numItems = trainset.n_items
trainingMatrix = np.zeros([numUsers, numItems, 10], dtype=np.float32)
for (uid, iid, rating) in trainset.all_ratings():
adjustedRating = int(float(rating) * 2.0) - 1
trainingMatrix[int(uid), int(iid), adjustedRating] = 1
#ユーザーごとにアイテムの可能な評価スコアに対して、ノードに2次元配列で平坦化する
trainingMatrix = np.reshape(trainingMatrix,
[trainingMatrix.shape[0], -1])
#可視化できるノードにアイテムの整数値と評価値を掛け算して、RBMを新たに構築する
rbm = RBM(trainingMatrix.shape[1],
hiddenDimensions=self.hiddenDim,
learningRate=self.learningRate,
batchSize=self.batchSize,
epochs=self.epochs)
rbm.Train(trainingMatrix)
self.predictedRatings = np.zeros([numUsers, numItems],
dtype=np.float32)
for uiid in range(trainset.n_users):
if (uiid % 50 == 0):
print("Processing user ", uiid)
recs = rbm.GetRecommendations([trainingMatrix[uiid]])
recs = np.reshape(recs, [numItems, 10])
for itemID, rec in enumerate(recs):
#評価値に最も高いスコアを付与して明示する
#rating = rec.argmax()
#5つ星に複数の評価値を導入する
#考えうるk値の正規化を行う
#予測する際に、期待値を取る
normalized = self.softmax(rec)
rating = np.average(np.arange(10), weights=normalized)
self.predictedRatings[uiid, itemID] = (rating + 1) * 0.5
return self
def fit(self, X, transform=False, load=False, name=None, save=True):
if load:
self.model = self.load()
else:
self.layer = []
X_input = X
input_size = X_input.shape[1]
self.input_shape = input_size
for i in range(self.n_layers):
output_size = self.params[i]
print(input_size)
self.layer.append(RBM(input_size, output_size))
input_size = output_size
self.layer[i].fit(X_input, self.n_iter[i])
X_input = self.layer[i].predict(X_input)
self.model = self.get_greed_model()
if save:
self.save(self.model)
if transform:
return self.predict(X)
def main(args):
train, test = read_data(args.in_dir)
train, val = split_data(train)
print('\n\nTrain: ', train.shape)
print('Val: ', val.shape)
print('Test: ', test.shape)
# RBM object
rbm = RBM(args.num_hidden, val.shape[1], args.lr, args.n, args.batch_size,
args.epochs)
# Train RBM
train_loss, val_loss = rbm.Train(train, val)
# Create output dir if it doesn't exist
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir)
# Plot error
plot_error(train_loss, val_loss, args.out_dir)
# Performance on Test set
error_test = rbm.reconstruction_error(test.T)
print("\n\n\nReconstruction error...\n")
print("Train : ", train_loss[-1])
print("Val : ", val_loss[-1])
print("Test : ", error_test)
# For viewing reconstruction
reconstruct_images(rbm, test, (args.image_height, args.image_width),
args.out_dir)
# Saving the model learned weights
pickle.dump([rbm.W, rbm.b_h, rbm.b_v],
open(args.out_dir + '\\weights.pkl', 'wb'))
print(f"\n\nRBM weights saved in {args.out_dir}\\weights.pkl")
MA_label = dataset['ylab']
MA_data, MA_label = shuffle(MA_data, MA_label)
#Normalize for unit variance and zero mean
MA_data = (MA_data - np.mean(MA_data)) / np.std(MA_data)
n_feature = np.shape(MA_data)[1]
n_hidden1 = int(feature2hiddenRatio * n_feature)
n_hidden2 = int(feature2hiddenRatio * n_hidden1)
n_hidden3 = int(feature2hiddenRatio * n_hidden2)
# n_hidden4 = int(feature2hiddenRatio * n_hidden3)
with tf.device("/gpu:0"):
rbm1 = RBM(n_hidden=n_hidden1,
n_visible=n_feature,
alpha=0.0001,
datatype="gaussian")
rbm2 = RBM(n_hidden=n_hidden2,
n_visible=n_hidden1,
alpha=0.0001,
datatype="binary")
rbm3 = RBM(n_hidden=n_hidden3,
n_visible=n_hidden2,
alpha=0.0001,
datatype="binary")
# rbm4 = RBM(n_hidden=n_hidden4, n_visible=n_hidden3, alpha=0.0001, datatype="binary")
for num in range(Epoch):
new_w, new_hb, new_vb, ReconErr = rbm1.train(MA_data)
print("Epoch: {}, Reconstruction Error: {}".format(num, ReconErr))
import torch
import numpy as np
import pandas as pd
import os
from RBM import RBM
from load_dataset import MNIST
if __name__ == '__main__':
mnist = MNIST()
train_x, train_y, test_x, test_y = mnist.load_dataset()
print('MAE for all 0 selection:', torch.mean(train_x))
vn = train_x.shape[1]
hn = 2500
rbm = RBM(vn, hn, epochs=100, mode='bernoulli', lr=0.0005, k=10, batch_size=128, gpu=True, optimizer='adam', savefile='mnist_trained_rbm.pt', early_stopping_patience=10)
rbm.train(train_x)
def initialize_variables(self):
# The DBN is an MLP, for which all weights of intermediate
# layers are shared with a different RBM. We will first
# construct the DBN as a deep multilayer perceptron, and when
# constructing each sigmoidal layer we also construct an RBM
# that shares weights with that layer. During pretraining we
# will train these RBMs (which will lead to chainging the
# weights of the MLP as well) During finetuning we will finish
# training the DBN by doing stochastic gradient descent on the
# MLP.
for i in range(self.n_layers):
# construct the sigmoidal layer
# the size of the input is either the number of hidden
# units of the layer below or the input size if we are on
# the first layer
if i == 0:
input_size = self.n_in
else:
input_size = self.hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of the
# hidden layer below or the input of the DBN if you are on
# the first layer
if i == 0:
layer_input = self.input
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=self.numpy_rng,
input=layer_input,
n_in=input_size,
n_out=self.hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
# its arguably a philosophical question... but we are
# going to only declare that the parameters of the
# sigmoid_layers are parameters of the DBN. The visible
# biases in the RBM are parameters of those RBMs, but not
# of the DBN.
self.params.extend(sigmoid_layer.params)
# Construct an RBM that shared weights with this layer
rbm_layer = RBM(numpy_rng=self.numpy_rng,
theano_rng=self.theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=self.hidden_layers_sizes[i],
W=sigmoid_layer.W,
hbias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
# We now need to add a logistic layer on top of the MLP
self.logistic_regression_layer = LogisticRegression(
input=self.sigmoid_layers[-1].output,
n_in=self.hidden_layers_sizes[-1],
n_out=self.n_out
)
self.params.extend(self.logistic_regression_layer.params)
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
num_epochs = 20000
Batch_size = 10
input_size = 28 * 28
hidden_size = 64
learning_rate = 0.1
tf.reset_default_graph()
X = tf.placeholder(tf.float32, [None, input_size], name="x")
rbm = RBM(input_size, hidden_size, X, learning_rate)
op = [rbm.W_up, rbm.c_up, rbm.b_up]
init = tf.global_variables_initializer()
#tf.default_graph().finalized()
with tf.Session() as sess:
sess.run(init)
for i in range(num_epochs):
weights = sess.run(
op, feed_dict={X: mnist.train.next_batch(Batch_size)[0]})
def main():
random.seed(0)
size = 40
rbm = RBM(size * size, 1000)
slp = 2
#for blob, params in randomBlobs(10, size, size):
# rbm.v = blob.flatten()
# plot(rbm, blob)
# sleep(slp)
#
# result = reshape(rbm.v, (size, size))
# plot(rbm, result)
# sleep(slp)
#
# rbm.v2h()
# plot(rbm, blob)
# sleep(slp)
#
# rbm.h2v()
# result = reshape(rbm.v, (size, size))
# plot(rbm, result)
# sleep(slp)
every = 2000
#m = log(.01/.1) / log(10000)
NN = 20001
bb = 1. / NN * log(.001 / 1.0)
elapsed = array([])
for ii in range(NN):
if ii % 100 == 0:
blob, params = randomBlobs(10, size, size).next()
#print params
#epsilon = .1 * (ii+1) ** m
#epsilon = .3 * exp(bb * ii)
epsilon = min(.1, 1.0 * exp(bb * ii))
#time0 = time()
rbm.learn1(blob.flatten(),
epsilon=epsilon,
activationH2V='gaussianReal',
param=1)
#elapsed = hstack((elapsed, time() - time0))
if ii % every == 0:
print '%d: epsilon is %f' % (ii, epsilon),
rbm.v = blob.flatten()
result = reshape(rbm.v, (size, size))
plot(rbm, result, 'Iteration %d' % ii, 'Data')
p.show()
sleep(.1) if ii == 0 else sleep(.1)
rbm.v2h()
rbm.h2v(activation='gaussianReal', param=0)
result = reshape(rbm.v, (size, size))
plot(rbm, result, 'Iteration %d' % ii, 'Reconstruction')
p.show()
sleep(.5) if ii == 0 else sleep(.5)
print 'mean of last 50 errors is', mean(rbm.reconErrorNorms[-50:])
os.makedirs(outf)
else:
print 'There exists a same model.'
exit()
logfile = os.path.join(outf, 'logfile.txt')
shutil.copy(os.path.realpath(__file__), os.path.join(outf, filename_script))
# data
assert (pixels[0] == 8)
train_dataset, validation_dataset, test_dataset = dataset.uci_digits_eval()
train_dataset = train_dataset.shuffle(shuffle_buffer).batch(batch_size)
dataset_iter = tfe.Iterator(train_dataset)
'''
model and loss
'''
rbm = RBM(vis_dim=vis_dim, hid_dim=hid_dim)
dec = SBNDEC1(dim_z2h, dim_h2v, temp=.1, hard=hard)
enc = ENC(dim_v2h, name='v2h', activation_fn=non, temp=.1, hard=False)
enc_h2z = ENC(dim_z2h[::-1],
name='z2h',
activation_fn=non,
temp=.1,
hard=False)
def loss_rbm(real_v):
_, real_h = enc.log_conditional_prob(real_v)
positve_phase = tf.reduce_mean(rbm.energy(real_h, real_v), axis=0)
_, _, fake_h, fake_v = dec.log_prob_all(batch_size)
negative_phase = tf.reduce_mean(rbm.energy(fake_h, fake_v), axis=0)
return positve_phase - negative_phase
mnist.test.labels
Nv = 784
v_shape = (28, 28)
Nh = 100
h1_shape = (10, 10)
batch_size = 100
epochs = 5
n_samples = mnist.train.num_examples
total_batch = int(n_samples / batch_size) * epochs
X = tf.placeholder("float", [None, 784])
with tf.variable_scope("l1"):
rbm1 = RBM(X, 100)
with tf.variable_scope("l2"):
rbm2 = RBM(rbm1.out_prob, 784, inbias=rbm1.b_out)
rbm2.create_rec()
rbm1.create_rec(rbm2.rec)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.assign(rbm2.W, tf.transpose(rbm1.W)))
for i in range(total_batch):
batch, label = mnist.train.next_batch(batch_size)
err, _ = sess.run([rbm1.mse, rbm1.train_op], feed_dict={X: batch})
if i % 100 == 0:
def __init__(self,
n_visible=6,
hidden_layer_sizes=[3, 3],
sample_copies=1,
sampler=None,
optimizer=None,
continuous_output=False,
verbose=0,
device=None):
'''Constructor for the class.
Arguments:
:param n_visible: The number nodes in the visible layer
:type n_visible: int
:param hidden_layer_sizes: The number nodes in each of
the hidden layers
:type hidden_layer_sizes: list of int
:param sample_copies: How many samples from a hidden layer are
drawn to train the next layer
:type sample_copies: int
:param sampler: Method used to draw samples from the model
:type sampler: :class:`samplers`
:param optimizer: Optimizer used for parameter updates
:type optimizer: :class:`optimizers`
:param continuous_output: Optional parameter to indicate whether
the visible layer is continuous-valued.
:type continuous_output: bool
:param verbose: Optional parameter to set verbosity mode
:type verbose: int
:param device: Device where to perform computations. None is CPU.
:type device: torch.device
'''
self.sample_copies = sample_copies
self.verbose = verbose
self.continuous_output = continuous_output
self.gen_layers = []
self.inference_layers = []
self.n_layers = len(hidden_layer_sizes)
assert self.n_layers > 0, 'You must specify at least one hidden layer'
if device is None:
device = torch.device('cpu')
self.device = device
if optimizer is None:
raise Exception('You must provide an appropriate optimizer')
self.optimizer = optimizer
if sampler is None:
raise Exception('You must provide an appropriate sampler')
self.sampler = sampler
# Construct DBN out of RBMs
for i in range(self.n_layers):
if i == 0:
input_size = n_visible
else:
input_size = hidden_layer_sizes[i - 1]
gen_layer = RBM(n_visible=input_size,
n_hidden=hidden_layer_sizes[i],
sampler=sampler,
optimizer=optimizer,
verbose=verbose,
device=device)
inf_layer = RBM(n_visible=input_size,
n_hidden=hidden_layer_sizes[i],
sampler=sampler,
optimizer=optimizer,
verbose=verbose,
device=device)
self.gen_layers.append(gen_layer)
self.inference_layers.append(inf_layer)
def main(args):
dbh = SQLHelper("database.sqlite")
rbm = RBM(visibleSize=784, hiddenSize=500)
cur = dbh.query("select * from weight where current=1;")
if len(cur) > 0:
if cur[0]['weight'] is not None:
params = cur[0]['hiddenBias'], cur[0]['visibleBias'], cur[0][
'weight']
rbm.setParams(params)
if len(sys.argv) > 1:
for arg in sys.argv:
if arg == '-t':
testSet = getTestSet(digits=[2])
if VISUALIZE:
# n = X[0] > np.random.rand(*X[0].shape)
# n = n.reshape(28,28)
n = testSet[0].reshape(28, 28)
strline = ""
for l in n:
for b in l:
if b:
strline += "*"
else:
strline += " "
strline += "\n"
print strline
sys.exit()
rbm.trainNetwork(testSet, dbh, 100)
# for i in xrange(10):
# testSet = getTestSet(digits=[i])
# if VISUALIZE:
# n = X[0] > np.random.rand(*X[0].shape)
# n = n.reshape(28,28)
# strline = ""
# for l in n:
# for b in l:
# if b:
# strline += "*"
# else:
# strline += " "
# strline += "\n"
# print strline
# sys.exit()
# rbm.trainNetwork(testSet, dbh, 100)
return
if arg == '-w':
#sqlite reg 830
i = 0
for weight in rbm.Weights:
retval = rbm.sigmoid(weight.copy())
saveImage((retval * 255),
"images/weight-" + str(i) + ".png")
i += 1
#testSet = getTestSet(randint(0,9))
for i in range(0, 10):
testSet = getTestSet(digits=[i],
Bernoulli=False,
dataSet='testing')
retval = testSet[random.randint(0, len(testSet))].copy()
saveImage(retval, "images/original-" + str(i) + ".png")
retval = np.array((np.random.rand(*retval.shape) < retval),
dtype=float)
retval = rbm.check(retval)
retval = retval.reshape(rbm.visibleLayerSize)
saveImage((255 * retval),
"images/result-" + str(i) + ".png")
return
return 0
