def test_mnist(learning_rate=0.1,
training_epochs=2,
dataset='../datasets/mnist.pkl.gz',
batch_size=20,
n_chains=20,
n_samples=10,
output_folder='MNIST_rbm_CD_plots',
n_hidden=500):
print 'Loading MNIST dataset...'
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
test_set_x, test_set_y = datasets[2]
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
index = T.lscalar()
x = T.matrix('x')
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2 ** 30))
print 'Creating RBM...'
rbm = RBM(input=x, n_visible=28 * 28,
n_hidden=n_hidden, numpy_rng=rng, theano_rng=theano_rng)
cost, updates = rbm.get_cost_updates(lr=learning_rate, k=15)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
os.chdir(output_folder)
train_rbm = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size]
},
name='train_rbm'
)
print 'Starting training with %d epochs' %training_epochs
plotting_time = 0.
start_time = timeit.default_timer()
for epoch in xrange(training_epochs):
mean_cost = []
for batch_index in xrange(n_train_batches):
mean_cost += [train_rbm(batch_index)]
print 'Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost)
plotting_start = timeit.default_timer()
image = Image.fromarray(
tile_raster_images(
X=rbm.W.get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=(10, 10),
tile_spacing=(1, 1)
)
)
image.save('filters_at_epoch_%i.png' % epoch)
plotting_stop = timeit.default_timer()
plotting_time += (plotting_stop - plotting_start)
end_time = timeit.default_timer()
pretraining_time = (end_time - start_time) - plotting_time
print ('Training took %.2f minutes' % (pretraining_time / 60.))
number_of_test_samples = test_set_x.get_value(borrow=True).shape[0]
test_idx = rng.randint(number_of_test_samples - n_chains)
persistent_vis_chain = theano.shared(
numpy.asarray(
test_set_x.get_value(borrow=True)[test_idx:test_idx + n_chains],
dtype=theano.config.floatX
)
)
plot_every = 1000
(
[
presig_hids,
hid_mfs,
hid_samples,
presig_vis,
vis_mfs,
vis_samples
],
updates
) = theano.scan(
rbm.gibbs_vhv,
outputs_info=[None, None, None, None, None, persistent_vis_chain],
n_steps=plot_every
)
updates.update({persistent_vis_chain: vis_samples[-1]})
sample_fn = theano.function(
[],
[
vis_mfs[-1],
vis_samples[-1]
],
updates=updates,
name='sample_fn'
)
image_data = numpy.zeros(
(29 * n_samples + 1, 29 * n_chains - 1),
dtype='uint8'
)
for idx in xrange(n_samples):
vis_mf, vis_sample = sample_fn()
print ' ... plotting sample ', idx
image_data[29 * idx:29 * idx + 28, :] = tile_raster_images(
X=vis_mf,
img_shape=(28, 28),
tile_shape=(1, n_chains),
tile_spacing=(1, 1)
)
image = Image.fromarray(image_data)
image.save('samples.png')
#--------------------
persistent_vis_chain_tse = theano.shared(
numpy.asarray(
test_set_x.get_value(borrow=True)[test_idx:test_idx + n_chains],
dtype=theano.config.floatX
)
)
plot_every = 10
(
[
presig_hids,
hid_mfs,
hid_samples,
presig_vis,
vis_mfs,
vis_samples
],
updates
) = theano.scan(
rbm.gibbs_vhv,
outputs_info=[None, None, None, None, None, persistent_vis_chain_tse],
n_steps=plot_every
)
updates.update({persistent_vis_chain_tse: vis_samples[-1]})
sample_hid_fn = theano.function(
[],
[
hid_mfs[-1],
hid_samples[-1]
],
updates=updates,
name='sample_hid_fn'
)
hid_mf, hid_sample = sample_hid_fn()
model_recon = manifold.TSNE(n_components=2, init='pca', random_state=0)
X_tsne=model_recon.fit_transform(hid_mf)
model_original = manifold.TSNE(n_components=2, init='pca', random_state=0)
X_tsne_original=model_original.fit_transform(test_set_x.get_value(borrow=True)[test_idx:test_idx + n_chains])
plot_embedding(X_tsne, test_set_y[test_idx:test_idx + n_chains].eval(),'Layer 1')
plt.savefig('original.png')
plot_embedding(X_tsne_original, test_set_y[test_idx:test_idx + n_chains].eval(),'Original')
plt.savefig('Tsne.png')
os.chdir('../')
load_path = 'wgan_model/best/dvd_electronics'
n_input = xs.shape[1]
n_hidden = [500]
X = tf.sparse_placeholder(dtype=tf.float32)
with tf.name_scope('generator'):
h1 = utils.fc_layer(X,
n_input,
n_hidden[0],
layer_name='hidden1',
input_type='sparse')
theta_G = [v for v in tf.global_variables() if 'hidden1' in v.name]
saver = tf.train.Saver(var_list=theta_G)
sess = tf.Session()
saver.restore(sess, load_path)
whole_xs_stt = utils.csr_2_sparse_tensor_tuple(xs)
whole_xt_stt = utils.csr_2_sparse_tensor_tuple(xt)
hs = sess.run(h1, feed_dict={X: whole_xs_stt})
ht = sess.run(h1, feed_dict={X: whole_xt_stt})
h_both = np.vstack([hs, ht])
y = np.hstack([ys, yt])
source_num = hs.shape[0]
tsne = TSNE(perplexity=30, n_components=2, n_iter=3300)
source_only_tsne = tsne.fit_transform(h_both)
utils.plot_embedding(source_only_tsne, y, source_num, 'wgan_tsne.pdf')
# Load 8x8 digits
digits = load_digits()
X = digits.data
y = digits.target
images = digits.images
# Plot digits
utils.plot_digits(X, 'A selection from the 64-dimensional (8x8) digits dataset')
#-------------------------------------------------------------------------------
# Part 1.1 - Random Projection
#-------------------------------------------------------------------------------
print('Computing random projection')
X_rp = SparseRandomProjection(n_components=2, random_state=42).fit_transform(X)
utils.plot_embedding(X_rp, y, images, 'Random projection of 8x8 digits')
#-------------------------------------------------------------------------------
# Part 1.2 - Principal Components Analysis (PCA)
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
# Part 1.2.1 - PCA from scratch
#-------------------------------------------------------------------------------
print('Computing PCA')
pca = lab10.PCA(n_components=2)
X_pca = pca.fit_transform(X)
utils.plot_explained_variance_ratio(pca.explained_variance_ratio_)
utils.plot_embedding(X_pca, y, images, 'PCA (your implementation) of 8x8 digits')
plt.plot(embeds[:, 0], embeds[:, 1], 'o', markersize=12)
plt.title('Perplexity {}'.format(i))
ax = plt.gca()
all_ims_ss = [all_ims_np[0]]
embeds_ss = [embeds[0:1]]
rnge = embeds[:, 0].max() - embeds[:, 0].min()
#and (embed[0] > (embeds[:, 0].min()) + 0.1*rnge) \
all_embeds_cat = np.concatenate(embeds_ss, 0)
for i, (im, embed) in enumerate(zip(all_ims_np, embeds)):
if (i % 2 == 0) \
and np.abs(embed - all_embeds_cat).sum(1).min() > 10:
all_ims_ss.append(im.transpose())
embeds_ss.append(np.expand_dims(embed, 0))
all_embeds_cat = np.concatenate(embeds_ss, 0)
ut.plot_embedding(embeds_ss, all_ims_ss, ax, sz=(320, 456))
plt.savefig(shared_path +
'scatter_plot_{}_perplexity_{}.png'.format(method, perp),
format='png')
plt.savefig('mouse_results/scatter_plot_{}_perplexity_{}.png'.format(
method, perp),
format='png')
else:
imap = mnf.Isomap()
def main():
classes = [i for i in range(SETTINGS['NUM_CLASSES'])]
training_batches = [
classes[i:i + SETTINGS['STEP_CLASSES']]
for i in range(0, len(classes), SETTINGS['STEP_CLASSES'])
]
SETTINGS['TRAINING_BATCHES'] = training_batches
checkpoint_path = os.path.join(
SETTINGS['CHECKPOINT_ROOT'], SETTINGS['DATASET'],
'StepClasses-{}'.format(str(SETTINGS['STEP_CLASSES'])),
'BufferSamples-{}'.format(str(SETTINGS['K_SHOT'])), SETTINGS['NET'],
SETTINGS['TIME_NOW'])
if not os.path.exists(checkpoint_path):
Path(checkpoint_path).mkdir(parents=True, exist_ok=True)
model_ckp_path = os.path.join(checkpoint_path,
'{net}-{idx}-{epoch}-{type}.pth')
save_setting(SETTINGS, checkpoint_path)
net = get_network(net=SETTINGS['NET'],
num_classes=SETTINGS['STEP_CLASSES'],
input_channels=3)
norm_alpha_loss = torch.nn.MSELoss()
norm_triangle_loss = torch.nn.MSELoss()
ce_criterion = torch.nn.CrossEntropyLoss()
zero_img = torch.zeros(size=[1, 3, 32, 32])
zero_label = torch.zeros(size=[1, 512])
old_classes = []
for iteration, training_sequence in enumerate(training_batches):
if not os.path.exists(
os.path.join(checkpoint_path, 'Plots', str(iteration))):
base_path = os.path.join(checkpoint_path, 'Plots', str(iteration))
base_gradients_path = os.path.join(base_path, 'Gradients')
g_zero_path = os.path.join(base_gradients_path, 'L_Zero')
g_alpha_path = os.path.join(base_gradients_path, 'L_Alpha')
g_triangle_path = os.path.join(base_gradients_path, 'L_Triangle')
loss_path = os.path.join(base_path, 'LossPlots')
embedding_path = os.path.join(base_path, 'EmbeddingPlots')
Path(loss_path).mkdir(parents=True, exist_ok=True)
Path(embedding_path).mkdir(parents=True, exist_ok=True)
Path(g_zero_path).mkdir(parents=True, exist_ok=True)
Path(g_alpha_path).mkdir(parents=True, exist_ok=True)
Path(g_triangle_path).mkdir(parents=True, exist_ok=True)
training_loader = get_train_loader(
SETTINGS['DATASET'],
accepted_class_labels=training_sequence,
norm_lambda=SETTINGS['NORM_LAMBDA'],
batch_size=SETTINGS['BATCH_SIZE'])
old_classes.extend(training_sequence)
test_loader = get_test_loader(SETTINGS['DATASET'],
accepted_class_labels=old_classes,
batch_size=5 * SETTINGS['BATCH_SIZE'])
if iteration == 0:
EPOCH = SETTINGS['STARTING_EPOCH']
lr = SETTINGS['STARTING_LEARNING_RATE']
else:
EPOCH = SETTINGS['OTHER_EPOCHS']
lr = SETTINGS['OTHER_LEARNING_RATE']
ce_optimizer = optim.SGD(params=net.parameters(), lr=lr, momentum=0.9)
triangle_optimizer = optim.SGD(params=net.parameters(),
lr=lr,
momentum=0.9)
zero_optimizer = optim.SGD(params=net.parameters(),
lr=lr,
momentum=0.9)
for epoch in range(EPOCH):
print('Processing iteration: {}\nEpoch:{}'.format(
iteration, epoch))
for batch_idx, data in enumerate(training_loader):
x, y, alpha, x2, y2, x_alpha, x_convex = data
y = y - iteration * SETTINGS['STEP_CLASSES']
if mode == MODE.SUPER_DEBUG:
print('---INPUT SHAPES---')
print(x.shape, y.shape, alpha.shape, x2.shape, y2.shape,
x_alpha.shape, x_convex.shape)
net.eval()
with torch.no_grad():
_, x2_features = net(x2.cuda())
_, alpha_x_features = net(x_alpha.cuda())
alpha_sq = torch.unsqueeze(alpha, dim=1)
if 'CE' in SETTINGS['LOSSES']:
net.train()
net.zero_grad()
preds, x_features = net(x.cuda())
l_ce = ce_criterion(preds, y.cuda())
l_ce.backward(retain_graph=True)
ce_gradients = get_gradient_magnitudes(net)
plot_gradients(ce_gradients, g_alpha_path,
'{}--{}'.format(epoch, batch_idx))
del ce_gradients
ce_optimizer.step()
else:
l_ce = DummyLoss()
"""net.train()
net.zero_grad()
_, x_features = net(x.cuda())
x_norm = torch.unsqueeze(torch.norm(x_features, p=2, dim=1), dim=1)
alpha_sq = torch.unsqueeze(alpha, dim=1)
alpha_x_norm = torch.unsqueeze(torch.norm(alpha_x_features, p=2, dim=1), dim=1)
# print(alpha_sq.shape, x_norm.shape, alpha_x_norm.shape)
l_a = norm_alpha_loss(x_norm*alpha_sq.cuda(), alpha_x_norm)
l_a.backward(retain_graph=True)
alpha_gradients = get_gradient_magnitudes(net)
plot_gradients(alpha_gradients, g_alpha_path, '{}--{}'.format(epoch, batch_idx))
del alpha_gradients
ce_optimizer.step()"""
if 'TRIANGLE' in SETTINGS['LOSSES']:
net.train()
net.zero_grad()
_, cvx_features = net(x_convex.cuda())
l_t = norm_triangle_loss(
torch.log(
torch.unsqueeze(torch.norm(cvx_features,
p=2,
dim=1),
dim=1)),
torch.log(
alpha_sq.cuda() * torch.unsqueeze(
torch.norm(x_features, p=2, dim=1), dim=1) +
(1 - alpha_sq.cuda()) * torch.unsqueeze(
torch.norm(x2_features, p=2, dim=1), dim=1)))
l_t.backward()
triangle_gradients = get_gradient_magnitudes(net)
plot_gradients(triangle_gradients, g_triangle_path,
'{}--{}'.format(epoch, batch_idx))
del triangle_gradients
triangle_optimizer.step()
else:
l_t = DummyLoss()
"""net.zero_grad()
_, zero_features = net(zero_img.cuda())
l_z = zero_loss(zero_features)/SETTINGS['BATCH_SIZE']
l_z.backward()
zero_gradients = get_gradient_magnitudes(net)
plot_gradients(zero_gradients, g_zero_path, '{}--{}'.format(epoch, batch_idx))
del zero_gradients
zero_optimizer.step()"""
plot_norm_losses(l_ce.item(),
l_t.item(),
0,
path=loss_path,
fid='Epoch:{}--BatchNo:{}'.format(
epoch, batch_idx))
train_acc = evaluate(net,
training_loader,
label_correction=iteration *
SETTINGS['STEP_CLASSES'])
print('Training accuracy: {}'.format(train_acc))
test_features = None
test_labels = None
for data in test_loader:
x_test, y_test, _, _, _, _, _ = data
net.eval()
with torch.no_grad():
_, x_test_features = net(x_test.cuda())
if test_features is None:
test_features = x_test_features.cpu()
test_labels = y_test.cpu()
else:
test_features = torch.cat(
[test_features,
x_test_features.cpu()], dim=0)
test_labels = torch.cat(
[test_labels, y_test.cpu()], dim=0)
plot_embedding(test_features.numpy(),
test_labels.numpy(),
num_classes=len(old_classes),
filepath=embedding_path,
filename='Epoch:{}'.format(epoch))
torch.save(
net.state_dict(),
model_ckp_path.format(net=SETTINGS['NET'],
idx=iteration,
epoch=epoch,
type='end'))
def visualize_results(self,
train_set_prob,
train_set_paths,
test_set_prob=None,
test_set_paths=None,
perplexity=15,
learning_rate=100,
image_size=42):
'''
Runs t-SNE on the given images and visualizes the result in a 2D plot.
If a test set is given (by passing a value for the test_set_prob
parameter), then the images of the test set are annotated with red
boxes.
Parameters
----------
train_set_prob : ndarray of float
An ndarray containing the probability vectors of each classified
image.
train_set_paths : list of str
A list containing the paths of the classified images.
test_set_prob : ndarray of float, default None
An ndarray containing the probability vectors of each classified
test image.
test_set_paths : list of str
A list containing the paths of the classified test images.
perplexity : int, default 15
The perplexity parameter for the t-SNE algorithm.
learning_rate : int, default 100
The learning rate parameter for the t-SNE algorithm.
image_size : int, default 42
The thumbnail size for the plot.
'''
model_tSNE = TSNE(n_components=2,
random_state=0,
learning_rate=learning_rate,
perplexity=perplexity)
if test_set_prob is not None:
x_tSNE = model_tSNE.fit_transform(
np.vstack((train_set_prob, test_set_prob)))
if type(train_set_paths) == list:
train_set_paths = np.array(train_set_paths)
if len(train_set_paths.shape) == 1:
train_set_paths = train_set_paths[:, np.newaxis]
if type(test_set_paths) == list:
test_set_paths = np.array(test_set_paths)
if len(test_set_paths.shape) == 1:
test_set_paths = test_set_paths[:, np.newaxis]
utils.plot_embedding(
x_tSNE,
np.squeeze(np.vstack((train_set_paths, test_set_paths))),
train_set_paths.shape[0] + test_set_paths.shape[0],
image_size=image_size,
test_image=True)
else:
x_tSNE = model_tSNE.fit_transform(train_set_prob)
utils.plot_embedding(x_tSNE,
train_set_paths,
len(train_set_paths),
image_size=image_size,
test_image=False)