def main():
"""
train and test the quality of the produced encodings by training a classifier using the encoded images
"""
args = parser.parse_args()
if args.cuda:
device = torch.device('cuda:0')
else:
device = torch.device('cpu')
data_dir = tools.select_data_dir()
transform = transforms.Compose([
transforms.ToTensor(),
])
trainset = torchvision.datasets.MNIST(root=data_dir,
train=True,
download=True,
transform=transform)
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True)
net = PixelCNN(n_channels=args.n_channels, kernel_size=args.kernel_size)
net.to(device)
if not args.skip_training:
optimizer = torch.optim.Adam(net.parameters(), lr=args.learning_rate)
for epoch in range(args.n_epochs):
for i, data in enumerate(trainloader, 0):
images, _ = data
images = images.to(device)
net.train()
optimizer.zero_grad()
y = net(images)
y = y.to(device)
loss = loss_fn(y, images)
loss = loss.to(device)
loss.backward()
optimizer.step()
with torch.no_grad():
samples = generate(net,
n_samples=args.n_samples,
device=device)
tools.plot_generated_samples(samples)
print('Train Epoch {}: Loss: {:.6f}'.format(
epoch + 1, loss.item()))
# Save the model to disk
tools.save_model(net, '10_pixelcnn.pth')
else:
net = PixelCNN(n_channels=args.n_channels,
kernel_size=args.kernel_size)
tools.load_model(net, '10_pixelcnn.pth', device)
# Generate samples
print('Generate samples with trained model')
with torch.no_grad():
samples = generate(net, n_samples=args.n_samples, device=device)
tools.plot_generated_samples(samples)
import os
import numpy as np
import matplotlib.pyplot as plt
from IPython import display
import torch
import torchvision
import torch.nn as nn
from torch.nn import functional as F
import torchvision.transforms as transforms
import torchvision.utils as utils
import tools
data_dir = tools.select_data_dir()
transform = transforms.Compose([
transforms.ToTensor(), # Transform to tensor
transforms.Normalize((0.5, ), (0.5, )) # Scale to [-1, 1]
])
trainset = torchvision.datasets.MNIST(root=data_dir,
train=True,
download=True,
transform=transform)
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=32,
shuffle=True)
def main():
"""
train and test the quality of the produced encodings by training a classifier using the encoded images
"""
skip_training = False
n_components = 10
n_epochs = 4
# device = torch.device('cuda:0')
device = torch.device('cpu')
data_dir = tools.select_data_dir()
transform = transforms.Compose([
transforms.ToTensor(), # Transform to tensor
transforms.Lambda(lambda x: x * torch.randn_like(x))
])
trainset = torchvision.datasets.MNIST(root=data_dir, train=True, download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=32, shuffle=True)
encoder = Encoder(n_components=n_components)
decoder = Decoder(n_components=n_components)
encoder = encoder.to(device)
decoder = decoder.to(device)
# Training loop
if not skip_training:
en_optimizer = torch.optim.Adam(encoder.parameters(),lr=0.001)
de_optimizer = torch.optim.Adam(decoder.parameters(),lr=0.001)
n_epochs = 10
for epoch in range(n_epochs):
for i, data in enumerate(trainloader, 0):
images, labels = data
images, labels = images.to(device), labels.to(device)
en_optimizer.zero_grad()
de_optimizer.zero_grad()
z_mu, z_logvar = encoder.forward(images)
sample = encoder.sample(z_mu,z_logvar)
y_mu, y_logvar = decoder.forward(sample)
loss =loss_kl(z_mu, z_logvar) + loss_loglik(y_mu, y_logvar, images)
loss.backward()
en_optimizer.step()
de_optimizer.step()
print('Train Epoch {}: Loss: {:.6f}'.format(epoch +1, loss.item()))
tools.save_model(encoder, 'vae_encoder.pth')
tools.save_model(decoder, 'vae_decoder.pth')
else:
encoder = Encoder(n_components=10)
tools.load_model(encoder, 'vae_encoder.pth', device)
decoder = Decoder(n_components=10)
tools.load_model(decoder, 'vae_decoder.pth', device)
# Test the quality of the produced embeddings by classification
print('start testing the quality of the produced embeddings by classification')
testset = torchvision.datasets.MNIST(root=data_dir, train=False, download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False)
traincodes, trainlabels = encode(trainset, encoder) # traincodes is (60000, 10)
testcodes, testlabels = encode(testset, encoder) # testcodes is (10000, 10)
# Train a simple linear classifier
logreg = LogisticRegression(C=1e5, solver='lbfgs', multi_class='multinomial', max_iter=400)
logreg.fit(traincodes.cpu(), trainlabels.cpu())
predicted_labels = logreg.predict(testcodes.cpu()) # (10000,)
# Compute accuracy of the linear classifier
accuracy = np.sum(testlabels.cpu().numpy() == predicted_labels) / predicted_labels.size
print('Accuracy with a linear classifier: %.2f%%' % (accuracy*100))
def main():
"""
train and test the quality of the produced encodings by training a classifier using the encoded images
"""
skip_training = False
n_components = 10
n_epochs = 4
# device = torch.device('cuda:0')
device = torch.device('cpu')
data_dir = tools.select_data_dir()
transform = transforms.Compose([
transforms.ToTensor(), # Transform to tensor
transforms.Normalize((0.5,), (0.5,)) # Minmax normalization to [-1, 1]
])
trainset = torchvision.datasets.MNIST(root=data_dir, train=True, download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=32, shuffle=True)
# Create a deep autoencoder
encoder = Encoder(n_components)
encoder.to(device)
decoder = Decoder(n_components)
decoder.to(device)
# Training loop
if not skip_training:
encoder_optimizer = torch.optim.Adam(encoder.parameters(),lr=0.001)
decoder_optimizer = torch.optim.Adam(decoder.parameters(),lr=0.001)
loss_method = nn.MSELoss()
for epoch in range(n_epochs):
for i, data in enumerate(trainloader, 0):
images, labels = data
encoder_optimizer.zero_grad()
decoder_optimizer.zero_grad()
encoder_output = encoder.forward(images)
decoder_output = decoder.forward(encoder_output)
loss = loss_method(decoder_output,images)
loss.backward()
encoder_optimizer.step()
decoder_optimizer.step()
print('Train Epoch {}: Loss: {:.6f}'.format(epoch +1, loss.item()))
print('training is finished.')
tools.save_model(encoder, 'ae_encoder.pth')
tools.save_model(decoder, 'ae_decoder.pth')
else:
device = torch.device("cpu")
encoder = Encoder(n_components=10)
tools.load_model(encoder, 'ae_encoder.pth', device)
decoder = Decoder(n_components=10)
tools.load_model(decoder, 'ae_decoder.pth', device)
# Test the quality of the produced embeddings by classification
print('start testing the quality of the produced embeddings by classification')
testset = torchvision.datasets.MNIST(root=data_dir, train=False, download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False)
traincodes, trainlabels = encode(trainset, encoder) # traincodes is (60000, 10)
testcodes, testlabels = encode(testset, encoder) # testcodes is (10000, 10)
logreg = LogisticRegression(C=1e5, solver='lbfgs', multi_class='multinomial')
logreg.fit(traincodes.cpu(), trainlabels.cpu())
predicted_labels = logreg.predict(testcodes.cpu()) # (10000,)
accuracy = np.sum(testlabels.cpu().numpy() == predicted_labels) / predicted_labels.size
print('Accuracy with a linear classifier: %.2f%%' % (accuracy*100))
def main():
device = torch.device('cuda:0')
n_features = 256
n_epochs = 40
batch_size = 64
skip_training = False
# Create the transformer model
encoder = Encoder(src_vocab_size=trainset.input_lang.n_words,
n_blocks=3,
n_features=n_features,
n_heads=16,
n_hidden=1024)
decoder = Decoder(tgt_vocab_size=trainset.output_lang.n_words,
n_blocks=3,
n_features=n_features,
n_heads=16,
n_hidden=1024)
encoder.to(device)
decoder.to(device)
# define training loop parameters
parameters = list(encoder.parameters()) + list(decoder.parameters())
adam = torch.optim.Adam(parameters, lr=0, betas=(0.9, 0.98), eps=1e-9)
optimizer = NoamOptimizer(n_features, 2, 10000, adam)
loss_method = nn.NLLLoss(ignore_index=0, reduction='mean')
# prepare data
data_dir = tools.select_data_dir()
trainset = TranslationDataset(data_dir, train=True)
trainloader = DataLoader(dataset=trainset,
batch_size=64,
shuffle=True,
collate_fn=collate,
pin_memory=True)
# training
if not skip_training:
for epoch in range(n_epochs):
loss = training_loop(encoder, decoder, optimizer, loss_method,
trainloader)
print(f'Train Epoch {epoch+1}: Loss: {loss}')
# save and load trained model
tools.save_model(encoder, 'tr_encoder.pth')
tools.save_model(decoder, 'tr_decoder.pth')
else:
encoder = Encoder(src_vocab_size=trainset.input_lang.n_words,
n_blocks=3,
n_features=256,
n_heads=16,
n_hidden=1024)
tools.load_model(encoder, 'tr_encoder.pth', device)
decoder = Decoder(tgt_vocab_size=trainset.output_lang.n_words,
n_blocks=3,
n_features=256,
n_heads=16,
n_hidden=1024)
tools.load_model(decoder, 'tr_decoder.pth', device)
# Generate translations with the trained model
# translate sentences from the training set
print('Translate training data:')
print('-----------------------------')
for i in range(5):
src_sentence, tgt_sentence = trainset[np.random.choice(len(trainset))]
print(
'>', ' '.join(trainset.input_lang.index2word[i.item()]
for i in src_sentence))
print(
'=', ' '.join(trainset.output_lang.index2word[i.item()]
for i in tgt_sentence))
out_sentence = translate(encoder, decoder, src_sentence)
print(
'<', ' '.join(trainset.output_lang.index2word[i.item()]
for i in out_sentence), '\n')
# translate sentences from the test set
testset = TranslationDataset(data_dir, train=False)
print('Translate test data:')
print('-----------------------------')
for i in range(5):
input_sentence, target_sentence = testset[np.random.choice(
len(testset))]
print(
'>', ' '.join(testset.input_lang.index2word[i.item()]
for i in input_sentence))
print(
'=', ' '.join(testset.output_lang.index2word[i.item()]
for i in target_sentence))
output_sentence = translate(encoder, decoder, input_sentence)
print(
'<', ' '.join(testset.output_lang.index2word[i.item()]
for i in output_sentence), '\n')
def main():
"""
train and test the quality of the produced encodings by training a classifier using the encoded images
"""
skip_training = False
n_components = 10
n_epochs = 4
# device = torch.device('cuda:0')
device = torch.device('cpu')
data_dir = tools.select_data_dir()
transform = transforms.Compose([
transforms.ToTensor(), # Transform to tensor
transforms.Lambda(lambda x: x * torch.randn_like(x))
])
trainset = torchvision.datasets.MNIST(root=data_dir,
train=True,
download=True,
transform=transform)
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=32,
shuffle=True)
dae = DAE(n_components)
dae.to(device)
# Training loop
if not skip_training:
optimizer = torch.optim.Adam(dae.parameters(), lr=0.001)
n_epochs = 5
loss_method = nn.MSELoss()
for epoch in range(n_epochs):
for i, data in enumerate(trainloader, 0):
images, _ = data
noise = torch.randn(*images.shape) * 0.2
noisy_images = images + noise
optimizer.zero_grad()
_, output = dae.forward(noisy_images)
loss = loss_method(output * noisy_images, images)
loss.backward()
optimizer.step()
print('Train Epoch {}: Loss: {:.6f}'.format(
epoch + 1, loss.item()))
tools.save_model(dae, 'dae.pth')
else:
device = torch.device('cpu')
dae = DAE(n_components=10)
tools.load_model(dae, 'dae.pth', device)
# Test the quality of the produced embeddings by classification
print(
'start testing the quality of the produced embeddings by classification'
)
testset = torchvision.datasets.MNIST(root=data_dir,
train=False,
download=True,
transform=transform)
testloader = torch.utils.data.DataLoader(testset,
batch_size=100,
shuffle=False)
traincodes, trainlabels = encode(trainset,
dae) # traincodes is (60000, 10)
testcodes, testlabels = encode(testset, dae) # testcodes is (10000, 10)
# Train a simple linear classifier
logreg = LogisticRegression(C=1e5,
solver='lbfgs',
multi_class='multinomial',
max_iter=200)
logreg.fit(traincodes.cpu(), trainlabels.cpu())
predicted_labels = logreg.predict(testcodes.cpu()) # (10000,)
accuracy = np.sum(
testlabels.cpu().numpy() == predicted_labels) / predicted_labels.size
print('Accuracy with a linear classifier: %.2f%%' % (accuracy * 100))
def main():
"""
"""
args = parser.parse_args()
if args.cuda:
device = torch.device("cuda:0")
else:
device = torch.device("cpu")
data_dir = tools.select_data_dir()
trainset = Sudoku(data_dir, train=True)
testset = Sudoku(data_dir, train=False)
trainloader = DataLoader(trainset, batch_size=args.batch_size, collate_fn=collate)
testloader = DataLoader(testset, batch_size=args.batch_size, collate_fn=collate)
# Create network
gnn = GNN(device)
if not args.skip_training:
optimizer = torch.optim.Adam(gnn.parameters(), lr=args.learning_rate)
loss_method = nn.CrossEntropyLoss(reduction="mean")
for epoch in range(args.n_epochs):
for i, data in enumerate(trainloader, 0):
inputs, targets, src_ids, dst_ids = data
inputs, targets = inputs.to(device), targets.to(device)
src_ids, dst_ids = src_ids.to(device), dst_ids.to(device)
optimizer.zero_grad()
gnn.zero_grad()
output = gnn.forward(inputs, src_ids, dst_ids)
output = output.to(device)
output = output.view(-1, output.shape[2])
targets = targets.repeat(7, 1)
targets = targets.view(-1)
loss = loss_method(output, targets)
loss.backward()
optimizer.step()
fraction = fraction_of_solved_puzzles(gnn, testloader, device)
print("Train Epoch {}: Loss: {:.6f} Fraction: {}".format(epoch + 1, loss.item(), fraction))
tools.save_model(gnn, "7_gnn.pth")
else:
gnn = GNN(device)
tools.load_model(gnn, "7_gnn.pth", device)
# Evaluate the trained model
# Get graph iterations for some test puzzles
with torch.no_grad():
inputs, targets, src_ids, dst_ids = iter(testloader).next()
inputs, targets = inputs.to(device), targets.to(device)
src_ids, dst_ids = src_ids.to(device), dst_ids.to(device)
batch_size = inputs.size(0) // 81
outputs = gnn(inputs, src_ids, dst_ids).to(device) # [n_iters, n_nodes, 9]
solution = outputs.view(gnn.n_iters, batch_size, 9, 9, 9).to(device)
final_solution = solution[-1].argmax(dim=3).to(device)
print("Solved puzzles in the current mini-batch:")
print((final_solution.view(-1, 81) == targets.view(batch_size, 81)).all(dim=1))
# Visualize graph iteration for one of the puzzles
ix = 0
for i in range(gnn.n_iters):
tools.draw_sudoku(solution[i, 0], logits=True)
fraction_solved = fraction_of_solved_puzzles(gnn, testloader,device)
print(f"Accuracy {fraction_solved}")