def evaluate(model, _config, _run):
training_config = _config['training']
cuda = _config['gpu']['cuda']
drop_rate = training_config['nx_drop_rate']
seed = training_config['seed']
# Create Class Specific Dataset
dataset = StandardDataset(
dataset='mnist',
data_dir='./data',
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=training_config['seed'])
eval_dataset = dataset.get_test_loader(bsize=1)
# Load Vision Class-specific AE movel
img_ae = get_ae(cuda)
# Get real and fake images from dataset
real_images = []
fake_images = []
with torch.no_grad():
for idx, data in enumerate(tqdm(eval_dataset)):
img = data[0]
symbol = F.one_hot(torch.tensor(data[1]), 10).float()
if cuda:
img = img.cuda()
symbol = symbol.cuda()
real_images.append(img[0])
# Encode Mod Latents
img_out, _ = model.generate([None, symbol])
nx_imgs = img_out[1]
fake_images.append(nx_imgs[0])
# Compute FID score
fid_score = e_utils.compute_fid(real=real_images, fake=fake_images, ae_model=img_ae, batch_size=64)
print("Rank = " + str(fid_score))
# Log values in mongodb
_run.log_scalar('Rank', fid_score)
# Save
with open(os.path.join(exp_dir_path('evaluation'), "fid_single_score_" + str(drop_rate)
+ "_" +str(seed) + ".txt"), 'w') as f:
print("Rank = " + str(fid_score), file=f)
with open(os.path.join(exp_dir_path('evaluation'), "fid_single_score_" + str(drop_rate)
+ "_" +str(seed) + ".pt"), 'wb') as f:
torch.save(fid_score, f)
ex.add_artifact(os.path.join(exp_dir_path('evaluation'),
"fid_single_score_" + str(drop_rate)
+ "_" +str(seed) + ".pt"), name="fid_single_score_" + str(drop_rate)
+ "_" +str(seed) + ".pt")
return
def evaluate(model, img_class, _config, _run):
results_dir = log_dir_path('evaluation')
training_config = _config['training']
eval_config = _config['evaluation']
cuda = _config['gpu']['cuda']
seed = training_config['seed']
# Create Dataset
dataset = StandardDataset(
dataset='mnist',
data_dir='./data',
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=training_config['seed'])
eval_dataset = dataset.get_test_loader(bsize=1)
# Setup training
n_samples = eval_config['eval_samples']
_run.log_scalar('eval_samples', n_samples)
# Evaluate recognition
img_res, sym_res = evaluate_recon(model, img_class, eval_dataset,
n_samples, cuda)
# Log values in mongodb
_run.log_scalar('Image Recon', img_res['recon'])
_run.log_scalar('Symbol to Image Recon', img_res['cm'])
_run.log_scalar('Symbol Recon', sym_res['recon'])
_run.log_scalar('Image to Symbol Recon', sym_res['cm'])
# Save
with open(os.path.join(exp_dir_path('evaluation'), "recon_res.txt"),
'w') as f:
print('\nFinal Results:\nImage Recon: ' + str(img_res['recon']) +
'\t' + '---> Symbol to Image Recon: ' + str(img_res['cm']) +
'\n' + 'Symbol Recon: ' + str(sym_res['recon']) + '\t' +
'---> Image to Symbol Recon: ' + str(sym_res['cm']) + '\n',
file=f)
with open(os.path.join(exp_dir_path('evaluation'), "img_recon_res.pt"),
'wb') as f:
torch.save(img_res, f)
ex.add_artifact(os.path.join(exp_dir_path('evaluation'),
"img_recon_res.pt"),
name='img_recon_res.pt')
with open(os.path.join(exp_dir_path('evaluation'), "sym_recon_res.pt"),
'wb') as f:
torch.save(sym_res, f)
ex.add_artifact(os.path.join(exp_dir_path('evaluation'),
"sym_recon_res.pt"),
name='sym_recon_res.pt')
return
def evaluate(model, _config, _run):
training_config = _config['training']
cuda = _config['gpu']['cuda']
# Create Dataset
dataset = StandardDataset(dataset='mnist',
data_dir='./data',
batch_size=training_config['batch_size'],
seed=training_config['seed'])
test_loader = dataset.test_loader
accuracy = evaluate_acc(model, test_loader, cuda=cuda)
_run.log_scalar('Test Accuracy', accuracy.item())
return
def train(_config, _run):
# Read configs
model_config = _config['model']
training_config = _config['training']
gpu_config = _config['gpu']
# Set seeds
torch.backends.cudnn.deterministic = True
torch.manual_seed(training_config['seed'])
np.random.seed(training_config['seed'])
random.seed(training_config['seed'])
torch.cuda.manual_seed(training_config['seed'])
# Create Model
nx_info, img_info, sym_info = t_utils.get_specs(model_config)
model = NexusModel(nx_info=nx_info,
img_info=img_info,
sym_info=sym_info,
use_cuda=gpu_config['cuda'])
# Create trainer
trainer = Trainer(model, training_config, gpu_config['cuda'])
# Create Dataset
dataset = StandardDataset(
dataset='fashion',
data_dir='./data',
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=training_config['seed'])
post_epoch_cb = PostEpochCb(model, dataset)
trainer.train(epochs=training_config['epochs'],
dataset=dataset,
cuda=gpu_config['cuda'],
post_epoch_cb=post_epoch_cb,
post_cb=post_cb)
def train(_config, _run):
# Read configs
training_config = _config['training']
gpu_config = _config['gpu']
device = torch.device("cuda" if _config['gpu']['cuda'] else "cpu")
results_dir = log_dir_path('results')
artifact_storage_interval = _config['model_debug'][
'artifact_storage_interval']
# Set seeds
torch.backends.cudnn.deterministic = True
torch.manual_seed(training_config['seed'])
np.random.seed(training_config['seed'])
random.seed(training_config['seed'])
torch.cuda.manual_seed(training_config['seed'])
# Create Classifier
model = MNIST_AE(b_dim=training_config['b_dim']).to(device)
epochs = training_config['epochs']
# Create Dataset
dataset = StandardDataset(dataset='mnist',
data_dir='./data',
batch_size=training_config['batch_size'],
seed=training_config['seed'])
train_loader, val_loader = dataset.train_loader, dataset.val_loader
test_loader = dataset.get_test_loader(bsize=20)
# Training objects
optimizer = torch.optim.Adam(model.parameters(),
lr=training_config['learning_rate'])
best_loss = sys.maxsize
for epoch in range(1, epochs + 1):
train_loss = train_epoch(model,
train_loader,
optimizer,
epoch,
cuda=True)
val_loss = test_epoch(model, val_loader, cuda=True)
_run.log_scalar('train_loss', train_loss)
_run.log_scalar('val_loss', val_loss)
# Best Loss
is_best = val_loss < best_loss
best_loss = min(val_loss, best_loss)
record_checkpoint(model=model,
loss=val_loss,
best_loss=best_loss,
optimizer=optimizer,
epoch=epoch,
is_best=is_best)
if epoch % artifact_storage_interval == 0:
# Data
with torch.no_grad():
model.eval()
data = next(iter(test_loader))[0].to(device)
# Generate modalities
image_out = model(data)
# Vision Recon
image_comp = torch.cat([
data.view(-1, 1, 28, 28).cpu(),
image_out.view(-1, 1, 28, 28).cpu()
])
torchvision.utils.save_image(
torchvision.utils.make_grid(image_comp,
padding=5,
pad_value=.5,
nrow=data.size(0)),
os.path.join(results_dir,
'mnist_ae_mod_e' + str(epoch) + '.png'))
ex.add_artifact(os.path.join(
results_dir, "mnist_ae_mod_e" + str(epoch) + '.png'),
name="mnist_ae_recon_e" + str(epoch) + '.png')
# Final Saving
ex.add_artifact(os.path.join(log_dir_path('trained_models'),
'mnist_ae_checkpoint.pth.tar'),
name='mnist_ae_last_checkpoint.pth.tar')
ex.add_artifact(os.path.join(log_dir_path('trained_models'),
'best_mnist_ae_model.pth.tar'),
name='best_mnist_ae_model.pth.tar')
def train(_config, _run):
# Read configs
training_config = _config['training']
gpu_config = _config['gpu']
device = torch.device("cuda" if _config['gpu']['cuda'] else "cpu")
# Set seeds
torch.backends.cudnn.deterministic = True
torch.manual_seed(training_config['seed'])
np.random.seed(training_config['seed'])
random.seed(training_config['seed'])
torch.cuda.manual_seed(training_config['seed'])
# Create Classifier
model = MNISTClassifier().to(device)
epochs = training_config['epochs']
# Create Dataset
dataset = StandardDataset(dataset='mnist',
data_dir='./data',
batch_size=training_config['batch_size'],
seed=training_config['seed'])
train_loader, test_loader = dataset.train_loader, dataset.val_loader
# Training objects
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=training_config['learning_rate'])
best_loss = sys.maxsize
for epoch in range(1, epochs + 1):
train_loss = train_epoch(model,
train_loader,
optimizer,
criterion,
epoch,
cuda=gpu_config)
val_loss, val_accuracy = test_epoch(model,
test_loader,
criterion,
cuda=gpu_config)
_run.log_scalar('train_loss', train_loss)
_run.log_scalar('val_loss', val_loss.item())
_run.log_scalar('val_accuracy', val_accuracy.item())
# Best Loss
is_best = val_loss < best_loss
best_loss = min(val_loss, best_loss)
record_checkpoint(model=model,
loss=val_loss,
best_loss=best_loss,
optimizer=optimizer,
epoch=epoch,
is_best=is_best)
# Final Saving
ex.add_artifact(os.path.join(log_dir_path('trained_models'),
'mnist_classifier_checkpoint.pth.tar'),
name='mnist_classifier_last_checkpoint.pth.tar')
ex.add_artifact(os.path.join(log_dir_path('trained_models'),
'best_mnist_classifier_model.pth.tar'),
name='best_mnist_classifier_model.pth.tar')