def sample(model, _config, _run):
model_eval_config = _config['generation']
cuda = _config['gpu']['cuda']
training_config = _config['training']
# Setup Generation
results_dir = exp_dir_path('sample_trajectories')
n_samples = model_eval_config['n_trajectory_samples']
# Create Dataset
dataset = MultimodalDataset(
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
modalities=['trajectory'],
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=training_config['seed'])
trj_norm = dataset.get_traj_normalization()
# Sample cross-modality from symbol
for i in tqdm(range(0, 10)):
sample_t = generate_class(model, i, n_samples, cuda)
# Trajectory
utils.save_trajectory(data=sample_t, path=results_dir, filename='image_class_' + str(i) + '.png',
traj_norm=trj_norm,
tmp_path=results_dir)
ex.add_artifact(os.path.join(results_dir, 'image_class_' + str(i) + '.png'),
name='image_class_' + str(i) + '.png')
def sample(model, _config, _run):
model_eval_config = _config['generation']
training_config = _config['training']
cuda = _config['gpu']['cuda']
# Setup Generation
results_dir = exp_dir_path('sample_sounds')
n_samples = model_eval_config['n_sound_samples']
# Create Dataset
dataset = MultimodalDataset(
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
modalities=['sound'],
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=training_config['seed'])
sound_norm = dataset.get_sound_normalization()
# Sample cross-modality from symbol
for i in tqdm(range(0, 10)):
samples_sound = generate_class(model, i, n_samples, cuda)
utils.save_sound(data=samples_sound,
path=results_dir,
filename="sound_class_" + str(i),
sound_norm=sound_norm)
def evaluate(model, _config, _run):
training_config = _config['training']
cuda = _config['gpu']['cuda']
drop_rate = training_config['nx_drop_rate']
seed = training_config['seed']
# Create Class Specific Dataset
dataset = MultimodalDataset(
modalities=['image'],
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=seed)
eval_dataset = dataset.get_test_loader(bsize=1)
# Load Vision Class-specific AE movel
img_ae = get_ae(cuda)
# Get real and fake images from dataset
real_images = []
fake_images = []
with torch.no_grad():
for idx, data in enumerate(tqdm(eval_dataset)):
img = data[1]
symbol = F.one_hot(torch.tensor(data[0]), 10).float()
if cuda:
img = img.cuda()
symbol = symbol.cuda()
real_images.append(img[0])
# Encode Mod Latents
img_out, _ = model.generate(x_s=symbol)
fake_images.append(img_out[1][0])
# Compute FID score
diss_score = e_utils.compute_fid(real=real_images, fake=fake_images, ae_model=img_ae, batch_size=64)
print("\n Rank score = " + str(diss_score))
print("\n")
# Log values in mongodb
_run.log_scalar('Rank score', diss_score)
# Save
with open(os.path.join(exp_dir_path('evaluation'), "diss_score_" + str(drop_rate)
+ "_" +str(seed) + ".txt"), 'w') as f:
print("Rank score = " + str(diss_score), file=f)
with open(os.path.join(exp_dir_path('evaluation'), "diss_score_" + str(drop_rate)
+ "_" +str(seed) + ".pt"), 'wb') as f:
torch.save(diss_score, f)
ex.add_artifact(os.path.join(exp_dir_path('evaluation'),
"diss_score_" + str(drop_rate)
+ "_" +str(seed) + ".pt"), name="diss_score_" + str(drop_rate)
+ "_" +str(seed) + ".pt")
return
def evaluate(model, img_class, _config, _run):
results_dir = log_dir_path('evaluation')
training_config = _config['training']
eval_config = _config['evaluation']
cuda = _config['gpu']['cuda']
seed = training_config['seed']
# Create Dataset
dataset = MultimodalDataset(
data_dir=exp_dir_path('../../../../scenarios/multimodal_dataset/data'),
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=seed)
eval_dataset = dataset.get_test_loader(bsize=1)
# Setup training
n_samples = eval_config['eval_samples']
_run.log_scalar('eval_samples', n_samples)
# Evaluate recognition
img_res, sym_res = evaluate_recon(model, img_class, eval_dataset,
n_samples, cuda)
# Log values in mongodb
_run.log_scalar('Image Recon', img_res['recon'])
_run.log_scalar('Image All Nexus Recon', img_res['all_nexus'])
_run.log_scalar('Image Nexus Recon', img_res['nexus'])
_run.log_scalar('Symbol to Image Recon', img_res['cm'])
_run.log_scalar('Symbol Recon', sym_res['recon'])
_run.log_scalar('Symbol All Nexus Recon', sym_res['all_nexus'])
_run.log_scalar('Symbol Nexus Recon', sym_res['nexus'])
_run.log_scalar('Image to Symbol Recon', sym_res['cm'])
# Save
with open(os.path.join(exp_dir_path('evaluation'), "img_recon_res.pt"),
'wb') as f:
torch.save(img_res, f)
ex.add_artifact(os.path.join(exp_dir_path('evaluation'),
"img_recon_res.pt"),
name='img_recon_res.pt')
with open(os.path.join(exp_dir_path('evaluation'), "sym_recon_res.pt"),
'wb') as f:
torch.save(sym_res, f)
ex.add_artifact(os.path.join(exp_dir_path('evaluation'),
"sym_recon_res.pt"),
name='sym_recon_res.pt')
return
def train(_config, _run):
# Read configs
model_config = _config['model']
training_config = _config['training']
gpu_config = _config['gpu']
# Set seeds
torch.backends.cudnn.deterministic = True
torch.manual_seed(training_config['seed'])
np.random.seed(training_config['seed'])
random.seed(training_config['seed'])
torch.cuda.manual_seed(training_config['seed'])
# Create Model
sound_info = t_utils.get_specs(model_config)
model = SigmaVAE(latent_dim=sound_info['mod_latent_dim'], use_cuda=gpu_config['cuda'])
# Create trainer
trainer = Trainer(model, training_config, gpu_config['cuda'])
# Create Dataset
dataset = MultimodalDataset(
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
modalities=["sound"],
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=training_config['seed'])
post_epoch_cb = PostEpochCb(model, dataset)
trainer.train(epochs=training_config['epochs'], dataset=dataset, cuda=gpu_config['cuda'],
post_epoch_cb=post_epoch_cb, post_cb=post_cb)
def evaluate(model, snd_class, _config, _run):
results_dir = log_dir_path('evaluation')
training_config = _config['training']
eval_config = _config['evaluation']
cuda = _config['gpu']['cuda']
drop_rate = training_config['nx_drop_rate']
seed = training_config['seed']
# Create Dataset
dataset = MultimodalDataset(
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=seed)
eval_dataset = dataset.get_test_loader(bsize=1)
# Setup training
n_samples = eval_config['eval_samples']
_run.log_scalar('eval_samples', n_samples)
# Evaluate sound recognition
snd_res = evaluate_recon(model, snd_class, eval_dataset, n_samples, cuda)
# Log values in mongodb
_run.log_scalar('acc single symbol', snd_res['single_symbol'])
_run.log_scalar('acc single image', snd_res['single_image'])
_run.log_scalar('acc single trajectory', snd_res['single_trajectory'])
_run.log_scalar('acc double symbol image', snd_res['double_symbol_image'])
_run.log_scalar('acc double symbol trajectory',
snd_res['double_symbol_trajectory'])
_run.log_scalar('acc double image trajectory',
snd_res['double_image_trajectory'])
_run.log_scalar('acc all mods', snd_res['all_mods'])
# Save
with open(
os.path.join(exp_dir_path('evaluation'), "snd_acc_" +
str(drop_rate) + "_" + str(seed) + ".pt"), 'wb') as f:
torch.save(snd_res, f)
ex.add_artifact(os.path.join(
exp_dir_path('evaluation'),
"snd_acc_" + str(drop_rate) + "_" + str(seed) + ".pt"),
name="snd_acc_" + str(drop_rate) + "_" + str(seed) + ".pt")
def train(_config, _run):
# Read configs
training_config = _config['training']
device = torch.device("cuda" if _config['gpu']['cuda'] else "cpu")
# Set seeds
torch.backends.cudnn.deterministic = True
torch.manual_seed(training_config['seed'])
np.random.seed(training_config['seed'])
random.seed(training_config['seed'])
torch.cuda.manual_seed(training_config['seed'])
# Create Classifier
model = Sound_Classifier().to(device)
epochs = training_config['epochs']
# Create Dataset
dataset = MultimodalDataset(
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
modalities=['sound'],
batch_size=training_config['batch_size'],
eval_samples=10,
validation_size=0.1,
seed=42)
train_loader, test_loader = dataset.train_loader, dataset.val_loader
# Training objects
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=training_config['learning_rate'])
best_loss = sys.maxsize
for epoch in range(1, epochs + 1):
train_loss = train_epoch(model, train_loader, optimizer, criterion, epoch, cuda=_config['gpu']['cuda'])
val_loss, val_accuracy = test_epoch(model, test_loader, criterion, cuda=_config['gpu']['cuda'])
_run.log_scalar('train_loss', train_loss)
_run.log_scalar('val_loss', val_loss.item())
_run.log_scalar('val_accuracy', val_accuracy.item())
# Best Loss
is_best = val_loss < best_loss
best_loss = min(val_loss, best_loss)
record_checkpoint(model=model, loss=val_loss, best_loss=best_loss,
optimizer=optimizer, epoch=epoch, is_best=is_best)
# Final Saving
ex.add_artifact(
os.path.join(
log_dir_path('trained_models'), 'sound_classifier_checkpoint.pth.tar'),
name='sound_classifier_last_checkpoint.pth.tar')
ex.add_artifact(
os.path.join(
log_dir_path('trained_models'), 'best_sound_classifier_model.pth.tar'),
name='best_sound_classifier_model.pth.tar')
def train(_config, _run):
# Read configs
model_config = _config['model']
training_config = _config['training']
gpu_config = _config['gpu']
# Set seeds
torch.backends.cudnn.deterministic = True
torch.manual_seed(training_config['seed'])
np.random.seed(training_config['seed'])
random.seed(training_config['seed'])
torch.cuda.manual_seed(training_config['seed'])
# Create Model
nx_info, img_info, trj_info, snd_info, sym_info = t_utils.get_specs(
model_config)
# Load sound vae
snd_vae = t_utils.load_sound_vae(gpu_config)
model = MVAEModel(nx_info=nx_info,
img_info=img_info,
trj_info=trj_info,
snd_info=snd_info,
sym_info=sym_info,
snd_vae=snd_vae,
use_cuda=gpu_config['cuda'])
# Create trainer
trainer = Trainer(model, training_config, gpu_config['cuda'])
# Create Dataset
dataset = MultimodalDataset(
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=training_config['seed'])
post_epoch_cb = PostEpochCb(model, dataset)
trainer.train(epochs=training_config['epochs'],
dataset=dataset,
cuda=gpu_config['cuda'],
post_epoch_cb=post_epoch_cb,
post_cb=post_cb)
def evaluate(model, _config, _run):
training_config = _config['training']
cuda = _config['gpu']['cuda']
# Create Dataset
dataset = MultimodalDataset(
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
modalities=['trajectory'],
batch_size=training_config['batch_size'],
eval_samples=10,
validation_size=0.1,
seed=42)
test_loader = dataset.test_loader
accuracy = evaluate_acc(model, test_loader, cuda=cuda)
_run.log_scalar('Test Accuracy', accuracy.item())
return
def evaluate(model, _config, _run):
training_config = _config['training']
cuda = _config['gpu']['cuda']
drop_rate = training_config['nx_drop_rate']
seed = training_config['seed']
# Create Class Specific Dataset
dataset = MultimodalDataset(
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
batch_size=training_config['batch_size'],
eval_samples=training_config['eval_samples'],
validation_size=training_config['validation_size'],
seed=seed)
eval_dataset = dataset.get_test_loader(bsize=1)
# Load Vision Class-specific AE movel
trj_ae = get_ae(cuda)
# Get real and fake trajectorys from dataset for every combination of modalities
# Single modality CMI
fake_trajectories_from_symbol = []
fake_trajectories_from_sound = []
fake_trajectories_from_image = []
real_trajectories = []
with torch.no_grad():
for idx, data in enumerate(tqdm(eval_dataset)):
img_data = data[1]
trj_data = data[2]
snd_data = data[3]
sym_data = torch.nn.functional.one_hot(data[0], num_classes=10).float()
if cuda:
img_data = img_data.cuda()
trj_data = trj_data.cuda()
snd_data = snd_data.cuda()
sym_data = sym_data.cuda()
real_trajectories.append(trj_data[0])
# Single mod CMI encoding
# From symbol
_, cm_trj, _, _ = model.generate(x_sym=sym_data)
fake_trajectories_from_symbol.append(cm_trj[1][0])
# From sound
_, cm_trj, _, _ = model.generate(x_snd=snd_data)
fake_trajectories_from_sound.append(cm_trj[1][0])
# From image
_, cm_trj, _, _ = model.generate(x_img=img_data)
fake_trajectories_from_image.append(cm_trj[1][0])
# Compute FID scores
trj_fid_scores_dic = {}
# SINGLE MOD CMI
# From Symbol
sym_fid_score = e_utils.compute_fid(real=real_trajectories, fake=fake_trajectories_from_symbol, ae_model=trj_ae, batch_size=64)
trj_fid_scores_dic['single_symbol'] = sym_fid_score
trj_fid_scores_dic['single_mod'] = [sym_fid_score]
# From Sound
snd_fid_score = e_utils.compute_fid(real=real_trajectories, fake=fake_trajectories_from_sound, ae_model=trj_ae, batch_size=64)
trj_fid_scores_dic['single_sound'] = snd_fid_score
trj_fid_scores_dic['single_mod'].append(snd_fid_score)
# From Trajectory
img_fid_score = e_utils.compute_fid(real=real_trajectories, fake=fake_trajectories_from_image, ae_model=trj_ae, batch_size=64)
trj_fid_scores_dic['single_image'] = img_fid_score
trj_fid_scores_dic['single_mod'].append(img_fid_score)
single_trj_dis_results = [trj_fid_scores_dic['single_symbol'],
trj_fid_scores_dic['single_image'],
trj_fid_scores_dic['single_sound']]
print("\n Trajectory Rank:")
print(" * Single Modality = " + str(np.mean(single_trj_dis_results))
+ " +-" + str(np.std(single_trj_dis_results)))
print("\n")
# Log values in mongodb
_run.log_scalar('fid single symbol', sym_fid_score)
_run.log_scalar('fid single sound', snd_fid_score)
_run.log_scalar('fid single image', img_fid_score)
# Save
with open(os.path.join(exp_dir_path('evaluation'), "trj_fid_" + str(drop_rate)
+ "_" +str(seed) + ".txt"), 'w') as f:
print(trj_fid_scores_dic, file=f)
with open(os.path.join(exp_dir_path('evaluation'), "trj_fid_" + str(drop_rate)
+ "_" +str(seed) + ".pt"), 'wb') as f:
torch.save(trj_fid_scores_dic, f)
ex.add_artifact(os.path.join(exp_dir_path('evaluation'),
"trj_fid_" + str(drop_rate)
+ "_" +str(seed) + ".pt"), name="trj_fid_" + str(drop_rate)
+ "_" +str(seed) + ".pt")
return
def train(_config, _run):
# Read configs
training_config = _config['training']
device = torch.device("cuda" if _config['gpu']['cuda'] else "cpu")
results_dir = log_dir_path('results')
artifact_storage_interval = _config['model_debug'][
'artifact_storage_interval']
# Set seeds
torch.backends.cudnn.deterministic = True
torch.manual_seed(training_config['seed'])
np.random.seed(training_config['seed'])
random.seed(training_config['seed'])
torch.cuda.manual_seed(training_config['seed'])
# Create Classifier
model = Sound_AE(b_dim=training_config['b_dim']).to(device)
epochs = training_config['epochs']
# Create Dataset
dataset = MultimodalDataset(
modalities=['sound'],
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
batch_size=training_config['batch_size'],
eval_samples=10,
validation_size=0.1,
seed=training_config['seed'])
sound_norm = dataset.get_sound_normalization()
train_loader, val_loader = dataset.train_loader, dataset.val_loader
test_loader = dataset.get_test_loader(bsize=20)
# Training objects
optimizer = torch.optim.Adam(model.parameters(),
lr=training_config['learning_rate'])
best_loss = sys.maxsize
for epoch in range(1, epochs + 1):
train_loss = train_epoch(model,
train_loader,
optimizer,
epoch,
cuda=_config['gpu']['cuda'])
val_loss = test_epoch(model, val_loader, cuda=_config['gpu']['cuda'])
_run.log_scalar('train_loss', train_loss)
_run.log_scalar('val_loss', val_loss)
# Best Loss
is_best = val_loss < best_loss
best_loss = min(val_loss, best_loss)
record_checkpoint(model=model,
loss=val_loss,
best_loss=best_loss,
optimizer=optimizer,
epoch=epoch,
is_best=is_best)
if epoch % artifact_storage_interval == 0:
# Data
with torch.no_grad():
model.eval()
data = next(iter(test_loader))[1].to(device)
# Generate modalities
out, _ = model(data)
# Trajectory
save_sound(data=data,
path=results_dir,
filename='og_sound_e' + str(epoch),
sound_norm=sound_norm)
for i in range(3):
ex.add_artifact(os.path.join(
results_dir,
"og_sound_e" + str(epoch) + '_' + str(i) + '.wav'),
name="og_sound_e" + str(epoch) + '_' +
str(i) + ".wav")
save_sound(data=out,
path=results_dir,
filename='sound_recon_e' + str(epoch),
sound_norm=sound_norm)
for i in range(3):
ex.add_artifact(os.path.join(
results_dir,
"sound_recon_e" + str(epoch) + '_' + str(i) + '.wav'),
name="sound_recon_e" + str(epoch) + '_' +
str(i) + ".wav")
# Final Saving
ex.add_artifact(os.path.join(log_dir_path('trained_models'),
'sound_ae_checkpoint.pth.tar'),
name='sound_ae_last_checkpoint.pth.tar')
ex.add_artifact(os.path.join(log_dir_path('trained_models'),
'best_sound_ae_model.pth.tar'),
name='best_sound_ae_model.pth.tar')
def train(_config, _run):
# Read configs
training_config = _config['training']
device = torch.device("cuda" if _config['gpu']['cuda'] else "cpu")
results_dir = log_dir_path('results')
artifact_storage_interval = _config['model_debug'][
'artifact_storage_interval']
# Set seeds
torch.backends.cudnn.deterministic = True
torch.manual_seed(training_config['seed'])
np.random.seed(training_config['seed'])
random.seed(training_config['seed'])
torch.cuda.manual_seed(training_config['seed'])
# Create Classifier
model = Image_AE(b_dim=training_config['b_dim']).to(device)
epochs = training_config['epochs']
# Create Dataset
dataset = MultimodalDataset(
modalities=['image'],
data_dir=exp_dir_path('../../../scenarios/multimodal_dataset/data'),
batch_size=training_config['batch_size'],
eval_samples=10,
validation_size=0.1,
seed=training_config['seed'])
train_loader, val_loader = dataset.train_loader, dataset.val_loader
test_loader = dataset.get_test_loader(bsize=20)
# Training objects
optimizer = torch.optim.Adam(model.parameters(), lr=training_config['learning_rate'])
best_loss = sys.maxsize
for epoch in range(1, epochs + 1):
train_loss = train_epoch(model, train_loader, optimizer, epoch, cuda=_config['gpu']['cuda'])
val_loss = test_epoch(model, val_loader, cuda=_config['gpu']['cuda'])
_run.log_scalar('train_loss', train_loss)
_run.log_scalar('val_loss', val_loss)
# Best Loss
is_best = val_loss < best_loss
best_loss = min(val_loss, best_loss)
record_checkpoint(model=model, loss=val_loss, best_loss=best_loss,
optimizer=optimizer, epoch=epoch, is_best=is_best)
if epoch % artifact_storage_interval == 0:
# Data
with torch.no_grad():
model.eval()
data = next(iter(test_loader))[1].to(device)
# Generate modalities
image_out = model(data)
# Vision Recon
image_comp = torch.cat([data.view(-1, 1, 28, 28).cpu(), image_out.view(-1, 1, 28, 28).cpu()])
torchvision.utils.save_image(torchvision.utils.make_grid(image_comp,
padding=5,
pad_value=.5,
nrow=data.size(0)),
os.path.join(results_dir, 'image_ae_mod_e' + str(epoch) + '.png'))
ex.add_artifact(os.path.join(results_dir, "image_ae_mod_e" + str(epoch) + '.png'),
name="image_ae_recon_e" + str(epoch) + '.png')
# Final Saving
ex.add_artifact(
os.path.join(
log_dir_path('trained_models'), 'image_ae_checkpoint.pth.tar'),
name='image_ae_last_checkpoint.pth.tar')
ex.add_artifact(
os.path.join(
log_dir_path('trained_models'), 'best_image_ae_model.pth.tar'),
name='best_image_ae_model.pth.tar')