def load_and_create_the_model(model_path):
loaded_state = torch.load(model_path)
epoch = loaded_state['epoch']
val_performance = loaded_state['val_performance']
args = loaded_state['args']
mean_tr = loaded_state['mean_tr']
std_tr = loaded_state['std_tr']
training_labels = loaded_state['training_labels']
model = LSTM_builder.BLSTMEncoder(num_layers=args.n_layers,
hidden_size=args.hidden_size,
embedding_depth=args.embedding_depth,
bidirectional=args.bidirectional,
dropout=args.dropout)
if torch.cuda.is_available():
model = nn.DataParallel(model).cuda()
else:
model = nn.DataParallel(model)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.learning_rate,
betas=(0.9, 0.999))
model.load_state_dict(loaded_state['model_state'])
optimizer.load_state_dict(loaded_state['optimizer_state'])
return (model, optimizer, epoch, val_performance, args, mean_tr, std_tr,
training_labels)
def convergence_of_LSTM(args):
visible_cuda_ids = ','.join(map(str, args.cuda_available_devices))
os.environ["CUDA_VISIBLE_DEVICES"] = visible_cuda_ids
(training_generator, mean_tr, std_tr, n_tr_batches) = \
fast_data_gen.get_data_generator(args,
return_stats=True)
val_args = copy.copy(args)
val_args.partition = 'val'
val_generator, n_val_batches = \
fast_data_gen.get_data_generator(val_args,
get_top=args.n_eval)
model = LSTM_enc.BLSTMEncoder(num_layers=args.n_layers,
hidden_size=args.hidden_size,
embedding_depth=args.embedding_depth,
bidirectional=args.bidirectional)
model = nn.DataParallel(model).cuda()
optimizer = torch.optim.Adam(model.parameters(),
lr=args.learning_rate,
betas=(0.9, 0.999))
k_means_obj = KMeans(n_clusters=2)
# just iterate over the data
history = {}
for epoch in np.arange(args.epochs):
train(args, model, training_generator, optimizer, mean_tr,
std_tr, epoch, history, n_tr_batches)
update_history.values_update([('loss', None)],
history,
update_mode='epoch')
if epoch % args.evaluate_per == 0:
eval(args, model, val_generator, mean_tr,
std_tr, epoch, history, n_val_batches, k_means_obj)
update_history.values_update([('sdr', None),
('sir', None),
('sar', None)],
history,
update_mode='epoch')
pprint(history['loss'][-1])
pprint(history['sdr'][-1])
pprint(history['sir'][-1])
pprint(history['sar'][-1])
print(
"BEST SDR: {}, SIR: {}, SAR {}".format(max(history['sdr']),
max(history['sir']),
max(history['sar'])))
def run_LSTM_experiment(args):
visible_cuda_ids = ','.join(map(str, args.cuda_available_devices))
os.environ["CUDA_VISIBLE_DEVICES"] = visible_cuda_ids
(training_generator, mean_tr, std_tr, n_tr_batches, n_tr_sources) =\
fast_data_gen.get_data_generator(args.train,
partition='train',
num_workers=args.num_workers,
return_stats=True,
get_top=args.n_train,
batch_size=args.batch_size,
return_n_batches=True,
labels_mask=args.training_labels,
return_n_sources=True)
val_generator, n_val_batches, n_val_sources = \
fast_data_gen.get_data_generator(args.val,
partition='val',
num_workers=args.num_workers,
return_stats=False,
get_top=args.n_val,
batch_size=args.batch_size,
return_n_batches=True,
labels_mask=None,
return_n_sources=True)
model = LSTM_enc.BLSTMEncoder(num_layers=args.n_layers,
hidden_size=args.hidden_size,
embedding_depth=args.embedding_depth,
bidirectional=args.bidirectional,
dropout=args.dropout)
if torch.cuda.is_available():
model = nn.DataParallel(model).cuda()
else:
model = nn.DataParallel(model)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.learning_rate,
betas=(0.9, 0.999))
assert n_val_sources == n_tr_sources, "Number of sources in both " \
"training and evaluation " \
"should be equal while " \
"training"
k_means_obj = KMeans(n_clusters=n_tr_sources)
# just iterate over the data
history = {}
for epoch in np.arange(args.epochs):
train(model, training_generator, optimizer, mean_tr,
std_tr, epoch, history, n_tr_batches, n_tr_sources,
training_labels=args.training_labels)
update_history.values_update([('loss', None)],
history,
update_mode='epoch')
# added the second term so it will save the model on the last epoch
if (epoch % args.eval_per == 0) or (epoch == (args.epochs - 1)):
eval(model, val_generator, mean_tr, std_tr, epoch,
history, n_val_batches, k_means_obj, n_val_sources,
args.batch_size)
update_history.values_update([('sdr', None),
('sir', None),
('sar', None)],
history,
update_mode='epoch')
# keep track of best performances so far
epoch_performance_dic = {
'sdr': history['sdr'][-1],
'sir': history['sir'][-1],
'sar': history['sar'][-1]
}
update_history.update_best_performance(
epoch_performance_dic, epoch, history,
buffer_size=args.save_best)
# save the model if it is one of the best according to SDR
if (history['sdr'][-1] >=
history['best_performances'][-1][0]['sdr']):
dataset_id = os.path.basename(args.train)
model_logger.save(model,
optimizer,
args,
epoch,
epoch_performance_dic,
dataset_id,
mean_tr,
std_tr,
training_labels=args.training_labels)
pprint(history['loss'][-1])
pprint(history['best_performances'])
def example_of_usage(args):
visible_cuda_ids = ','.join(map(str, args.cuda_available_devices))
os.environ["CUDA_VISIBLE_DEVICES"] = visible_cuda_ids
print(visible_cuda_ids)
print(torch.cuda.current_device())
training_generator, n_batches = data_generator.get_data_generator(args)
timing_dic = {}
before = time.time()
model = LSTM_enc.BLSTMEncoder(num_layers=args.n_layers,
hidden_size=args.hidden_size,
embedding_depth=args.embedding_depth,
bidirectional=args.bidirectional)
timing_dic['Iitializing model'] = time.time() - before
model = model.cuda()
timing_dic['Transfering model to device'] = time.time() - before
optimizer = torch.optim.Adam(model.parameters(),
lr=args.learning_rate,
betas=(0.9, 0.999))
# just iterate over the data
epochs = 10
for epoch in np.arange(epochs):
print("Training for epoch: {}...".format(epoch))
for batch_data in training_generator:
(abs_tfs, real_tfs, imag_tfs, duet_masks, ground_truth_masks,
sources_raw, amplitudes, n_sources) = batch_data
input_tfs, index_ys = abs_tfs.cuda(), duet_masks.cuda()
# the input sequence is determined by time and not freqs
# before: input_tfs = batch_size x (n_fft/2+1) x n_timesteps
input_tfs = input_tfs.permute(0, 2, 1).contiguous()
index_ys = index_ys.permute(0, 2, 1).contiguous()
one_hot_ys = converters.one_hot_3Dmasks(index_ys, n_sources[0])
timing_dic = {}
optimizer.zero_grad()
vs = model(input_tfs)
before = time.time()
flatened_ys = one_hot_ys.view(one_hot_ys.size(0), -1,
one_hot_ys.size(-1)).cuda()
naive_loss = affinity_losses.naive(vs, flatened_ys)
naive_loss.backward()
optimizer.step()
now = time.time()
print("Naive Loss: {}".format(naive_loss))
timing_dic['Naive Loss Implementation Time'] = now - before
optimizer.zero_grad()
vs = model(input_tfs)
before = time.time()
expanded_vs = vs.view(vs.size(0), one_hot_ys.size(1),
one_hot_ys.size(2), vs.size(-1)).cuda()
diagonal_loss = affinity_losses.diagonal(expanded_vs, one_hot_ys)
diagonal_loss.backward()
optimizer.step()
now = time.time()
print("Diagonal Loss: {}".format(diagonal_loss))
timing_dic['Diagonal Loss Implementation Time'] = now - before
pprint(timing_dic)