def main(_):
pp.pprint(flags.FLAGS.__flags)
if not os.path.exists(FLAGS.checkpoint_dir):
os.makedirs(FLAGS.checkpoint_dir)
if not os.path.exists(FLAGS.sample_dir):
os.makedirs(FLAGS.sample_dir)
if not os.path.exists(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
config = tf.ConfigProto(
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9),
device_count = {'GPU': 1},
allow_soft_placement=True
#log_device_placement=True,
)
config.device_filters.append('/gpu:0')
config.device_filters.append('/cpu:0')
with tf.Session(config=config) as sess:
#with tf.device('/gpu:0'):
autoencoder = Autoencoder(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size,
dataset_name=FLAGS.dataset, noise = FLAGS.noise, is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir)
if FLAGS.is_train:
autoencoder.train(FLAGS)
elif FLAGS.is_run:
autoencoder.run(FLAGS)
else:
autoencoder.load(FLAGS.checkpoint_dir)
def main():
parser = argparse.ArgumentParser(
description='Simple training script for training model')
parser.add_argument(
'--epochs', help='Number of epochs (default: 75)', type=int, default=75)
parser.add_argument(
'--batch-size', help='Batch size of the data (default: 16)', type=int, default=16)
parser.add_argument(
'--learning-rate', help='Learning rate (default: 0.001)', type=float, default=0.001)
parser.add_argument(
'--seed', help='Random seed (default:1)', type=int, default=1)
parser.add_argument(
'--data-path', help='Path for the downloaded dataset (default: ../dataset/)', default='../dataset/')
parser.add_argument(
'--dataset', help='Dataset name. Must be one of MNIST, STL10, CIFAR10')
parser.add_argument(
'--use-cuda', help='CUDA usage (default: False)', type=bool, default=False)
parser.add_argument(
'--network-type', help='Type of the network layers. Must be one of Conv, FC (default: FC)', default='FC')
parser.add_argument(
'--weight-decay', help='weight decay (L2 penalty) (default: 1e-5)', type=float, default=1e-5)
parser.add_argument(
'--log-interval', help='No of batches to wait before logging training status (default: 50)', type=int, default=50)
parser.add_argument(
'--save-model', help='For saving the current model (default: True)', type=bool, default=True)
args = parser.parse_args()
epochs = args.epochs # number of epochs
batch_size = args.batch_size # batch size
learning_rate = args.learning_rate # learning rate
torch.manual_seed(args.seed) # seed value
# Creating dataset path if it doesn't exist
if args.data_path is None:
raise ValueError('Must provide dataset path')
else:
data_path = args.data_path
if not os.path.isdir(data_path):
os.mkdir(data_path)
# Downloading proper dataset and creating data loader
if args.dataset == 'MNIST':
T = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
train_data = torchvision.datasets.MNIST(
data_path, train=True, download=True, transform=T)
test_data = torchvision.datasets.MNIST(
data_path, train=False, download=True, transform=T)
ip_dim = 1 * 28 * 28 # input dimension
h1_dim = int(ip_dim / 2) # hidden layer 1 dimension
op_dim = int(ip_dim / 4) # output dimension
elif args.dataset == 'STL10':
T = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
train_data = torchvision.datasets.STL10(
data_path, split='train', download=True, transform=T)
test_data = torchvision.datasets.STL10(
data_path, split='test', download=True, transform=T)
ip_dim = 3 * 96 * 96 # input dimension
h1_dim = int(ip_dim / 2) # hidden layer 1 dimension
op_dim = int(ip_dim / 4) # output dimension
elif args.dataset == 'CIFAR10':
T = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
train_data = torchvision.datasets.CIFAR10(
data_path, train=True, download=True, transform=T)
test_data = torchvision.datasets.CIFAR10(
data_path, train=False, download=True, transform=T)
ip_dim = 3 * 32 * 32 # input dimension
h1_dim = int(ip_dim / 2) # hidden layer 1 dimension
op_dim = int(ip_dim / 4) # output dimension
elif args.dataset is None:
raise ValueError('Must provide dataset')
else:
raise ValueError('Dataset name must be MNIST, STL10 or CIFAR10')
train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False)
# use CUDA or not
device = 'cpu'
if args.use_cuda is False:
if torch.cuda.is_available():
warnings.warn(
'CUDA is available, please use for faster convergence')
else:
device = 'cpu'
else:
if torch.cuda.is_available():
device = 'cuda'
else:
raise ValueError('CUDA is not available, please set it False')
# Type of layer
if args.network_type == 'FC':
auto_encoder = Autoencoder(ip_dim, h1_dim, op_dim).to(device)
elif args.network_type == 'Conv':
auto_encoder = ConvolutionAE().to(device)
else:
raise ValueError('Network type must be either FC or Conv type')
# Train the model
auto_encoder.train()
criterion = torch.nn.MSELoss()
optimizer = torch.optim.Adam(
lr=learning_rate, params=auto_encoder.parameters(), weight_decay=args.weight_decay)
for n_epoch in range(epochs): # loop over the dataset multiple times
reconstruction_loss = 0.0
for batch_idx, (X, Y) in enumerate(train_loader):
X = X.view(X.size()[0], -1)
X = Variable(X).to(device)
encoded, decoded = auto_encoder(X)
optimizer.zero_grad()
loss = criterion(X, decoded)
loss.backward()
optimizer.step()
reconstruction_loss += loss.item()
if (batch_idx + 1) % args.log_interval == 0:
print('[%d, %5d] Reconstruction loss: %.5f' %
(n_epoch + 1, batch_idx + 1, reconstruction_loss / args.log_interval))
reconstruction_loss = 0.0
if args.save_model:
torch.save(auto_encoder.state_dict(), "Autoencoder.pth")
# Save real images
data_iter = iter(test_loader)
images, labels = data_iter.next()
torchvision.utils.save_image(torchvision.utils.make_grid(
images, nrow=4), 'images/actual_img.jpeg')
# Load trained model and get decoded images
auto_encoder.load_state_dict(torch.load('Autoencoder.pth'))
auto_encoder.eval()
images = images.view(images.size()[0], -1)
images = Variable(images).to(device)
encoded, decoded = auto_encoder(images)
# Save decoded images
if args.dataset == 'MNIST':
decoded = decoded.view(decoded.size()[0], 1, 28, 28)
elif args.dataset == 'STL10':
decoded = decoded.view(decoded.size()[0], 3, 96, 96)
elif args.dataset == 'CIFAR10':
decoded = decoded.view(decoded.size()[0], 3, 32, 32)
torchvision.utils.save_image(torchvision.utils.make_grid(
decoded, nrow=4), 'images/decoded_img.jpeg')
def main(**kwargs):
"""
Main function that trains the model.
1. Retrieve arguments from kwargs
2. Prepare MNIST
3. Train
4. Display first batch of test set
Args:
add_noise: Whether to add noise (DAE) to input image or not (AE).
binarize_input: Whether to binarize input image pixels to 0 and 1.
epochs: How many epochs to train model.
loss: Which loss function to use. Either cross-entropy or mean square error.
lr: Learning rate.
latent_dim: Dimension of latent variable.
print_every: How often to print training progress.
"""
# Retrieve arguments
add_noise = kwargs.get('add_noise', defaults['add_noise'])
binarize_input = kwargs.get('binarize_input', defaults['binarize_input'])
epochs = kwargs.get('epochs', defaults['epochs'])
loss = kwargs.get('loss', defaults['loss'])
lr = kwargs.get('learning_rate', defaults['learning_rate'])
latent_dim = kwargs.get('latent_dim', defaults['latent_dim'])
print_every = kwargs.get('print_every', defaults['print_every'])
# Load and transform MNIST dataset
if binarize_input:
trsf = transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: (x >= 0.5).float())
])
else:
trsf = transforms.ToTensor()
MNIST_train = datasets.MNIST(root='MNIST',
train=True,
transform=trsf,
download=True)
MNIST_test = datasets.MNIST(root='MNIST',
train=False,
transform=trsf,
download=True)
# Create dataloader
train_loader = torch.utils.data.DataLoader(MNIST_train,
batch_size=64,
shuffle=True)
test_loader = torch.utils.data.DataLoader(MNIST_test,
batch_size=64,
shuffle=False)
# Create model and optimizer
autoencoder = Autoencoder(latent_dim=latent_dim).to(device)
optimizer = optim.Adam(autoencoder.parameters(), lr=lr)
# Select loss function
criterion = CE_criterion if loss == 'CE' else MSE_criterion
# Train
autoencoder.train()
for epoch in range(epochs):
for batch_ind, (input_data, _) in enumerate(train_loader):
input_data = input_data.to(device)
# Forward propagation
if add_noise:
noised_input_data = F.dropout(input_data, p=0.5)
output = autoencoder(noised_input_data)
else:
output = autoencoder(input_data)
# Calculate loss
loss = criterion(output, input_data)
# Backward propagation
optimizer.zero_grad()
loss.backward()
# Update parameters
optimizer.step()
# Print progress
if batch_ind % print_every == 0:
train_log = 'Epoch {:2d}/{:2d}\tLoss: {:.6f}\tTrain: [{}/{} ({:.0f}%)] '.format(
epoch + 1, epochs,
loss.cpu().item(), batch_ind + 1, len(train_loader),
100. * batch_ind / len(train_loader))
print(train_log, end='\r')
sys.stdout.flush()
# Learning rate decay
if epoch == 4:
optimizer = optim.Adam(autoencoder.parameters(), lr=lr / 10)
# Display training result with test set
with torch.no_grad():
images, _ = iter(test_loader).next()
images = images.to(device)
if add_noise:
noise_images = F.dropout(images, p=0.5)
denoised_output = autoencoder(noise_images)
output = autoencoder(images)
display_batch("Binarized truth" if binarize_input else "Truth",
images, binarize_input)
display_batch("Truth with noise", noise_images, binarize_input)
display_batch("Reconstruction of noised image", output,
binarize_input)
display_batch("Reconstruction of clean image", denoised_output,
binarize_input)
else:
output = autoencoder(images)
display_batch("Binarized truth" if binarize_input else "Truth",
images, binarize_input)
display_batch("Reconstruction", output, binarize_input)
outputs_path = "outputs"
if not os.path.exists(outputs_path):
os.mkdir(outputs_path)
# TODO Make some checkpoints work.
#best_model_path = str(DATA_DIR / f'outputs/{MODEL_CKPT_FILENAME}')
#checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
# filepath=best_model_path,
# monitor="val_loss",
# save_best_only=True,
# verbose=1
#)
training_callbacks = [AzureLogCallback(run)]
# Train the model.
model.train(dataset_train,
dataset_validate,
dataset_anomaly,
epochs=CONFIG.EPOCHS,
batch_size=CONFIG.BATCH_SIZE,
shuffle_buffer_size=CONFIG.SHUFFLE_BUFFER_SIZE,
render=CONFIG.RENDER,
render_every=5,
callbacks=training_callbacks,
outputs_path=outputs_path
#kl_loss_factor=CONFIG.KL_LOSS_FACTOR
)
# Done.
run.complete()
time_step_count = 0
if i % (report_every * 10) == 0:
# print()
# print("REPORTING")
# print()
model.eval()
with torch.no_grad():
total_loss = 0.
count = 0
for data in data_stream(valid_filenames[:20],
shuffle=False,
batch_size=32):
# get the inputs
input = torch.from_numpy(data.astype(
np.float32)).cuda()
loss = model(input)
# print(loss)
total_loss += loss.data.item()
count += 1
valid_loss = total_loss / count
if valid_loss < best_loss:
print("Best valid loss:", valid_loss)
with open('model.pt', 'wb') as f:
torch.save(model, f)
best_loss = valid_loss
else:
print("Valid loss:", valid_loss)
random.shuffle(valid_filenames)
scheduler.step(valid_loss)
model.train()
return {'loss': loss, 'bce': bce, 'l1': l1, 'w2': w2, 'encode': z, 'decode': recon_x}
mnist = torch.utils.data.DataLoader(datasets.MNIST("./mnist/", train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor()
])), batch_size=128, shuffle=True)
cudnn.benchmark = True
ae = Autoencoder().cuda()
print(ae)
optimizer = torch.optim.Adam(ae.parameters())
total_epoch = 50
trainer = SAE(ae, optimizer, random_uniform, num_projections=25)
ae.train()
for epoch in range(total_epoch):
for index, (img, label) in enumerate(mnist):
img = img.cuda()
#img = img.expand(img.data.shape[0], 3, 28, 28)
batch_result = trainer.train(img)
if (index+1) % 10 == 0:
print("loss: {:.4f} \t l1:{:.4f} \t bce:{:.4f} \t w2:{:.4f}".format(
batch_result["loss"], batch_result["l1"], batch_result["bce"], batch_result["w2"]))
shuffle=True)
autoencoder = Autoencoder(args.input_size,
args.output_size,
args.hidden_size,
bn=False)
optimizer = optim.Adam(autoencoder.parameters(), lr=0.0002)
criterion = nn.BCELoss()
hmc = Hamiltonian(autoencoder.decoder, args.output_size, 0.1,
args.num_steps_in_leap, args.num_samples)
if args.cuda:
autoencoder = autoencoder.cuda()
for epoch in range(1, args.epochs + 1):
autoencoder.train()
train_loss = 0
mnist_data = list(iter(train_loader))
for batch_idx in range(0, 1000):
data = torch.FloatTensor(mnist_data[batch_idx][0])
data = Variable(data)
if args.cuda:
data = data.cuda()
optimizer.zero_grad()
recon_batch, mu, logvar, encoded_rep = autoencoder(data)
if args.hmc:
init_x = encoded_rep.data.cpu().numpy()
hmc.get_hmc_sample(init_x, data, gpu=args.cuda)
optimizer.step()