def run_average_precision():
models = ['lstm', 'mlp', 'logistic']
num_experiments = 5
results = {}
evaluator = Evaluator()
for model in models:
print('Model: ', model)
average_precision = 0
for i in range(num_experiments):
average_precision += run(args, model_name=model, report_results=False, training_steps=2000)
results[model.upper()] = average_precision/num_experiments
evaluator.save_model_comparison(results, file_path='./saved/precision/precision_comparison.png')
def build(configuration, device_configuration, experiments_path,
experiment_name, results_path):
data_stream = VCTKFeaturesStream('../data/vctk', configuration,
device_configuration.gpu_ids,
device_configuration.use_cuda)
if configuration['decoder_type'] == 'deconvolutional':
vqvae_model = ConvolutionalVQVAE(configuration,
device_configuration.device).to(
device_configuration.device)
evaluator = Evaluator(device_configuration.device, vqvae_model,
data_stream, configuration, results_path,
experiment_name)
else:
raise NotImplementedError(
"Decoder type '{}' isn't implemented for now".format(
configuration['decoder_type']))
if configuration['trainer_type'] == 'convolutional':
trainer = ConvolutionalTrainer(device_configuration.device,
data_stream, configuration,
experiments_path, experiment_name,
**{'model': vqvae_model})
else:
raise NotImplementedError(
"Trainer type '{}' isn't implemented for now".format(
configuration['trainer_type']))
vqvae_model = nn.DataParallel(
vqvae_model, device_ids=device_configuration.gpu_ids
) if device_configuration.use_data_parallel else vqvae_model
return trainer, evaluator
def load(experiments_path,
experiment_name,
results_path,
data_path='../data'):
error_caught = False
try:
configuration_file, checkpoint_files = PipelineFactory.load_configuration_and_checkpoints(
experiments_path, experiment_name)
except:
ConsoleLogger.error(
'Failed to load existing configuration. Building a new model...'
)
error_caught = True
# Load the configuration file
ConsoleLogger.status('Loading the configuration file')
configuration = None
with open(experiments_path + os.sep + configuration_file, 'r') as file:
configuration = yaml.load(file, Loader=yaml.FullLoader)
device_configuration = DeviceConfiguration.load_from_configuration(
configuration)
if error_caught or len(checkpoint_files) == 0:
trainer, evaluator = PipelineFactory.build(configuration,
device_configuration,
experiments_path,
experiment_name,
results_path)
else:
latest_checkpoint_file, latest_epoch = CheckpointUtils.search_latest_checkpoint_file(
checkpoint_files)
# Update the epoch number to begin with for the future training
configuration['start_epoch'] = latest_epoch
configuration['num_epochs'] = 60
#latest_checkpoint_file = 'baseline_15_checkpoint.pth'
#print(latest_checkpoint_file)
# Load the checkpoint file
checkpoint_path = experiments_path + os.sep + latest_checkpoint_file
ConsoleLogger.status(
"Loading the checkpoint file '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path,
map_location=device_configuration.device)
# Load the data stream
ConsoleLogger.status('Loading the data stream')
data_stream = VCTKFeaturesStream('/atlas/u/xuyilun/vctk',
configuration,
device_configuration.gpu_ids,
device_configuration.use_cuda)
def load_state_dicts(model, checkpoint, model_name,
optimizer_name):
# Load the state dict from the checkpoint to the model
model.load_state_dict(checkpoint[model_name])
# Create an Adam optimizer using the model parameters
optimizer = optim.Adam(model.parameters())
# Load the state dict from the checkpoint to the optimizer
optimizer.load_state_dict(checkpoint[optimizer_name])
# Map the optimizer memory into the specified device
for state in optimizer.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
state[k] = v.to(device_configuration.device)
return model, optimizer
# If the decoder type is a deconvolutional
if configuration['decoder_type'] == 'deconvolutional':
# Create the model and map it to the specified device
vqvae_model = ConvolutionalVQVAE(
configuration, device_configuration.device).to(
device_configuration.device)
evaluator = Evaluator(device_configuration.device, vqvae_model,
data_stream, configuration, results_path,
experiment_name)
# Load the model and optimizer state dicts
vqvae_model, vqvae_optimizer = load_state_dicts(
vqvae_model, checkpoint, 'model', 'optimizer')
elif configuration['decoder_type'] == 'wavenet':
vqvae_model = WaveNetVQVAE(configuration,
data_stream.speaker_dic,
device_configuration.device).to(
device_configuration.device)
evaluator = Evaluator(device_configuration.device, vqvae_model,
data_stream, configuration, results_path,
experiment_name)
# Load the model and optimizer state dicts
vqvae_model, vqvae_optimizer = load_state_dicts(
vqvae_model, checkpoint, 'model', 'optimizer')
else:
raise NotImplementedError(
"Decoder type '{}' isn't implemented for now".format(
configuration['decoder_type']))
# Temporary backward compatibility
if 'trainer_type' not in configuration:
ConsoleLogger.error(
"trainer_type was not found in configuration file. Use 'convolutional' by default."
)
configuration['trainer_type'] = 'convolutional'
if configuration['trainer_type'] == 'convolutional':
trainer = ConvolutionalTrainer(
device_configuration.device, data_stream, configuration,
experiments_path, experiment_name, **{
'model': vqvae_model,
'optimizer': vqvae_optimizer
})
else:
raise NotImplementedError(
"Trainer type '{}' isn't implemented for now".format(
configuration['trainer_type']))
# Use data parallelization if needed and available
vqvae_model = vqvae_model
return trainer, evaluator, configuration, device_configuration
def run(args, model_name='logistic', report_result=True, epoc=2000):
dataset_info = dataset.get_train_test_val_data()
train_data = dataset_info[0]
train_label = dataset_info[1]
train_seqlen = dataset_info[2]
test_data = dataset_info[3]
test_label = dataset_info[4]
test_seqlen = dataset_info[5]
val_data = dataset_info[6]
val_label = dataset_info[7]
val_seqlen = dataset_info[8]
vocab_size = dataset_info[9]
num_classes = dataset.get_number_classes()
model_class = get_model(model_name)
model = model_class(max_seq_len, state_size, vocab_size, num_classes)
tf.reset_default_graph()
model.build_model()
loss, optimizer = model.step_training(learning_rate=learning_rate)
val_precission = 0.0
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=1)
ckpt = None
if args.save_model_path:
ckpt = tf.train.get_checkpoint_state(args.save_model_path)
if ckpt and ckpt.model_checkpoint_path and restore_model:
saver.restore(sess, ckpt.model_checkpoint_path)
training_precision, validation_precision, train_loss = [], [], []
for step in range(epoc):
batch_x, batch_y, batch_seqlen = dataset.next_train_balanced(
batch_size=batch_size)
sess.run(optimizer,
feed_dict={
model.x: batch_x,
model.y: batch_y,
model.batch_size: len(batch_seqlen)
})
if step % display_step == 0 and step > 0:
ops = [model.precision, loss]
precision, loss_ = sess.run(ops,
feed_dict={
model.x: batch_x,
model.y: batch_y,
model.batch_size:
len(batch_seqlen)
})
print("Step " + str(step) + ", Minibatch Loss= " + \
"{:.6f}".format(loss_) + ", Training Precision= " + \
"{:.5f}".format(precision))
training_precision.append((step, precision))
train_loss.append((step, loss_))
if step % validation_step == 0 and step > 0:
val_precision, y_probs = sess.run(
[model.precision, model.probs],
feed_dict={
model.x: val_data,
model.y: val_label,
model.batch_size: len(batch_seqlen)
})
print("Step " + str(step) +
", Validation Precision= {:.5f}".format(val_precision))
validation_precision.append((step, val_precision))
if saver and args.save_model_path:
saver.save(sess,
os.path.join(args.save_model_path,
'trained_model.ckpt'),
global_step=step)
# report results during training:
if report_results:
evaluator = Evaluator()
class_names = dataset.get_class_names()
y_actual = val_label
y_preds = np.argmax(y_probs, axis=1)
evaluator.save_confussion_matrix(
y_actual,
y_preds,
class_names,
normalize=True,
file_path=
'./saved/confussion_matrix/confussion_matrix_%s.png' %
(model_name),
title='Confussion matrix for %s at time %d' %
(model_name.upper(), step))
evaluator.save_precision(
training_precision,
validation_precision,
file_path='./saved/precision/precision_%s.png' %
(model_name),
title='%s model at step %d' %
(model_name.upper(), step))
evaluator.save_loss(
train_loss,
file_path='./saved/loss/loss_training_%s.png' %
(model_name),
title='%s model at step %d' %
(model_name.upper(), step))
return val_precision
def run(args, model_name='lstm', report_results=True, training_steps=2000):
'''
Run method that drives the logic of the program
:param args: external arguments needed to run
:param model_name: name of model (i.e., lstm, logistic, mlp)
:param report_results: wether to plot current results or not
:param training_steps: number of training steps
:return:
'''
dataset_info = dataset.get_train_test_val_data()
#[size_test, max_utterance_length] = (3067, 2080)
train_data = dataset_info[0]
train_label = dataset_info[1]
train_seqlen = dataset_info[2]
test_data = dataset_info[3]
test_label = dataset_info[4]
test_seqlen = dataset_info[5]
val_data = dataset_info[6]
val_label = dataset_info[7]
val_seqlen = dataset_info[8]
vocab_size = dataset_info[9]
model_class = get_model(model_name)
model = model_class(max_seq_len, state_size, vocab_size, dataset.get_number_classes())
tf.reset_default_graph()
model.build_model() #se constuye el modelo para que se cree la arquitectura
loss, optimizer = model.step_training(learning_rate=learning_rate) #se pasa el modelo a proceso de optimización(función que permite reducir el loss/perdidad/error)
val_precision = 0.0
#para entrenar un modelo en tf hay que crear una session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer()) #se toman las variables globales inicializadas anteriormente
saver = tf.train.Saver(max_to_keep=1)
ckpt = None
if args.save_model_path:
ckpt = tf.train.get_checkpoint_state(args.save_model_path)
if ckpt and ckpt.model_checkpoint_path and restore_model:
saver.restore(sess, ckpt.model_checkpoint_path)
training_precision, validation_precision, train_loss = [], [], []
#este modelo se va a entrenar la cantidad declarada en training_steps
for step in range(training_steps):
batch_x, batch_y, batch_seqlen = dataset.next_train_balanced(batch_size=batch_size)
#se llama al optimizer para reducir el error
sess.run(optimizer, feed_dict={model.x: batch_x,
model.y: batch_y,
model.batch_size: len(batch_seqlen)})
if step % display_step == 0 and step > 0:
ops = [model.precision, loss]
precision, loss_ = sess.run(ops, feed_dict={model.x: batch_x,
model.y: batch_y,
model.batch_size: len(batch_seqlen)})
print("Step " + str(step) + ", Minibatch Loss= " + \
"{:.6f}".format(loss_) + ", Training Precision= " + \
"{:.5f}".format(precision))
training_precision.append((step, precision))
train_loss.append((step, loss_))
if step % validation_step == 0 and step > 0:
val_precision, y_probs = sess.run([model.precision, model.probs], feed_dict={model.x: val_data,
model.y: val_label,
model.batch_size: len(batch_seqlen)})
print("Step " + str(step) + ", Validation Precision= {:.5f}".format(val_precision))
validation_precision.append((step, val_precision))
if saver and args.save_model_path:
saver.save(sess, os.path.join(args.save_model_path, 'trained_model.ckpt'),
global_step = step)
# report results during training:
# se guarda los plots de la matriz de confución, y la grafica de precisión y perdida
if report_results:
evaluator = Evaluator()
class_names = dataset.get_class_names()
y_actual = val_label
y_preds = np.argmax(y_probs, axis=1)
evaluator.save_confussion_matrix(y_actual, y_preds, class_names, normalize=True,
file_path='./saved/confussion_matrix/confussion_matrix_%s.png' %(model_name),
title='Confussion matrix for %s at time %d' %(model_name.upper(), step))
evaluator.save_precision(training_precision, validation_precision,
file_path='./saved/precision/precision_%s.png' %(model_name),
title='%s model at step %d' %(model_name.upper(), step))
evaluator.save_loss(train_loss, file_path='./saved/loss/loss_training_%s.png' % (model_name),
title='%s model at step %d' % (model_name.upper(), step))
return val_precision