def train(self):
# fix z for visualization
n_fixed_samples = self.val_block_size * self.val_block_size
fixed_z = np.random.uniform(-1.0,
1.0,
size=(n_fixed_samples, self.z_dim))
# get dataset
mnist_dataset = get_mnist_by_name(self.batch_size, self.dataset_type)
# setup tracking variables
steps = 0
losses = []
start_time = time.time()
with tf.Session() as sess:
# reset tensorflow variables
sess.run(tf.global_variables_initializer())
# start training
for e in range(self.epochs):
# setup dataset iterator for graph mode
iterator = mnist_dataset.make_one_shot_iterator()
next_elem = iterator.get_next()
while True:
try:
self.train_step(sess, next_elem, steps, losses)
steps += 1
except tf.errors.OutOfRangeError:
print('End of dataset')
break
# save generation results at every n epochs
if e % self.save_every == 0:
self.save_generator_output(sess, e, fixed_z)
self.saver.save(
sess,
self.ckpt_fn,
global_step=tf.train.get_or_create_global_step())
# save final output
self.save_generator_output(sess, e, fixed_z)
self.saver.save(sess,
self.ckpt_fn,
global_step=tf.train.get_or_create_global_step())
end_time = time.time()
elapsed_time = end_time - start_time
# save losses as image
losses_fn = os.path.join(
self.assets_dir,
'{:s}-{:s}-losses.png'.format(self.dataset_type,
self.gan_loss_type))
utils.save_losses(losses, ['Discriminator', 'Generator'], elapsed_time,
losses_fn)
return
def callback_fn(self, params):
print("Function value: ", end='')
loss = objective_function(params, self.X, self.y, self.lambd)
print(loss)
print("Average gradient: ", end='')
avg_grad = np.mean(
d_optimization_function(params, self.X, self.y, self.lambd)**2)
print(avg_grad)
print()
self.iters += 1
save_params(params, self.filename, self.iters)
save_losses(loss, self.loss_filename, self.iters)
def main(data_dir, plane, epochs, lr, weight_decay, device=None):
diagnoses = ['abnormal', 'acl', 'meniscus']
exp = f'{datetime.now():%Y-%m-%d_%H-%M}'
out_dir, losses_path = create_output_dir(exp, plane)
if device is None:
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('Creating data loaders...')
train_loader = make_data_loader(data_dir,
'train',
plane,
device,
shuffle=True)
valid_loader = make_data_loader(data_dir, 'valid', plane, device)
print(f'Creating models...')
# Create a model for each diagnosis
models = [MRNet().to(device), MRNet().to(device), MRNet().to(device)]
# Calculate loss weights based on the prevalences in train set
pos_weights = calculate_weights(data_dir, 'train', device)
criterions = [nn.BCEWithLogitsLoss(pos_weight=weight) \
for weight in pos_weights]
optimizers = [make_adam_optimizer(model, lr, weight_decay) \
for model in models]
lr_schedulers = [make_lr_scheduler(optimizer) for optimizer in optimizers]
min_valid_losses = [np.inf, np.inf, np.inf]
print(f'Training a model using {plane} series...')
print(f'Checkpoints and losses will be save to {out_dir}')
for epoch, _ in enumerate(range(epochs), 1):
print(f'=== Epoch {epoch}/{epochs} ===')
batch_train_losses = np.array([0.0, 0.0, 0.0])
batch_valid_losses = np.array([0.0, 0.0, 0.0])
for inputs, labels in train_loader:
inputs, labels = inputs.to(device), labels.to(device)
batch_loss = batch_forward_backprop(models, inputs, labels,
criterions, optimizers)
batch_train_losses += batch_loss
valid_preds = []
valid_labels = []
for inputs, labels in valid_loader:
inputs, labels = inputs.to(device), labels.to(device)
batch_preds, batch_loss = \
batch_forward(models, inputs, labels, criterions)
batch_valid_losses += batch_loss
valid_labels.append(labels.detach().cpu().numpy().squeeze())
valid_preds.append(batch_preds)
batch_train_losses /= len(train_loader)
batch_valid_losses /= len(valid_loader)
print_stats(batch_train_losses, batch_valid_losses, valid_labels,
valid_preds)
save_losses(batch_train_losses, batch_valid_losses, losses_path)
update_lr_schedulers(lr_schedulers, batch_valid_losses)
for i, (batch_v_loss, min_v_loss) in \
enumerate(zip(batch_valid_losses, min_valid_losses)):
if batch_v_loss < min_v_loss:
save_checkpoint(epoch, plane, diagnoses[i], models[i],
optimizers[i], out_dir)
min_valid_losses[i] = batch_v_loss
scheduler1.step()
scheduler2.step()
if confs['loss'] == 'wae-gan':
scheduler1_disc.step()
scheduler2_disc.step()
train_loss, recon, match = train(model, trainloader, optimizer, confs,
epoch)
train_losses.append(train_loss)
train_loss_data['total'].append(train_loss)
train_loss_data['recon'].append(recon)
train_loss_data['match'].append(match)
if testloader != None:
test_loss, recon, match = test(model,
testloader,
epoch,
confs,
add_noise=confs['noise'])
test_losses.append(test_loss)
test_loss_data['total'].append(train_loss)
test_loss_data['recon'].append(recon)
test_loss_data['match'].append(match)
print('Epoch: {} Train Loss: {} Test Loss: {}'.format(
epoch, train_loss, test_loss))
else:
print('Epoch: {} Train Loss: {}'.format(epoch, train_loss))
if epoch % config.SAVEFREQ == 0:
save_model(confs['dataset'] + '_' + confs['type'], model, epoch, confs)
save_losses(train_loss_data, confs, epoch, 'train')
save_losses(test_loss_data, confs, epoch, 'test')
save_model(confs['dataset'], model, epoch, confs)
def train(self):
n_fixed_samples = self.val_block_size * self.val_block_size
fixed_z = np.random.uniform(-1, 1, size=(n_fixed_samples, self.z_dim))
fixed_y = np.zeros(shape=[n_fixed_samples, self.y_dim])
for s in range(n_fixed_samples):
loc = s % self.y_dim
fixed_y[s, loc] = 1
steps = 0
losses = []
start_time = time.time()
with tf.Session() as sess:
# reset tensorflow variables
sess.run(tf.global_variables_initializer())
# start training
for e in range(self.epochs):
for ii in range(self.mnist_loader.train.num_examples //
self.batch_size):
batch_x, batch_y = self.mnist_loader.train.next_batch(
self.batch_size)
# rescale images to -1 ~ 1
batch_x = np.reshape(batch_x, (-1, 28, 28, 1))
batch_x = batch_x * 2.0 - 1.0
# Sample random noise for G
batch_z = np.random.uniform(-1,
1,
size=(self.batch_size,
self.z_dim))
fd = {
self.inputs_x: batch_x,
self.inputs_y: batch_y,
self.inputs_z: batch_z
}
# Run optimizers
_ = sess.run(self.d_opt, feed_dict=fd)
_ = sess.run(self.g_opt, feed_dict=fd)
_ = sess.run(self.ac_opt, feed_dict=fd)
# print losses
if steps % self.print_every == 0:
# At the end of each epoch, get the losses and print them out
train_loss_d = self.d_loss.eval(fd)
train_loss_g = self.g_loss.eval(fd)
train_loss_ac = self.ac_loss.eval(fd)
print(
"Epoch {}/{}...".format(e + 1, self.epochs),
"Discriminator Loss: {:.4f}...".format(
train_loss_d),
"Generator Loss: {:.4f}...".format(train_loss_g),
"Auxilary Classifier Loss: {:.4f}...".format(
train_loss_ac))
losses.append(
(train_loss_d, train_loss_g, train_loss_ac))
steps += 1
# save generation results at every epochs
if e % self.save_every == 0:
val_out = sess.run(network.generator(self.inputs_z,
y=self.inputs_y,
reuse=True,
is_training=False),
feed_dict={
self.inputs_y: fixed_y,
self.inputs_z: fixed_z
})
image_fn = os.path.join(
self.assets_dir,
'{:s}-val-e{:03d}.png'.format(self.dataset_type,
e + 1))
utils.validation(val_out,
self.val_block_size,
image_fn,
color_mode='L')
end_time = time.time()
elapsed_time = end_time - start_time
# save losses as image
losses_fn = os.path.join(self.assets_dir,
'{:s}-losses.png'.format(self.dataset_type))
utils.save_losses(losses, ['Discriminator', 'Generator', 'Auxilary'],
elapsed_time, losses_fn)
return
def train(self):
val_size = self.val_block_size * self.val_block_size
steps = 0
losses = []
new_epochs = self.d_train_freq * self.epochs
start_time = time.time()
with tf.Session() as sess:
# reset tensorflow variables
sess.run(tf.global_variables_initializer())
# start training
for e in range(new_epochs):
for ii in range(self.mnist_loader.train.num_examples //
self.batch_size):
# no need labels
batch_x, _ = self.mnist_loader.train.next_batch(
self.batch_size)
# rescale images to -1 ~ 1
batch_x = np.reshape(batch_x, (-1, 28, 28, 1))
batch_x = batch_x * 2.0 - 1.0
# Sample random noise for G
batch_z = np.random.uniform(-1,
1,
size=(self.batch_size,
self.z_dim))
fd = {self.inputs_x: batch_x, self.inputs_z: batch_z}
# Run optimizers (train D more than G)
_ = sess.run(self.d_weight_clip)
_ = sess.run(self.d_opt, feed_dict=fd)
if ii % self.d_train_freq == 0:
_ = sess.run(self.g_opt, feed_dict=fd)
# print losses
if steps % self.print_every == 0:
# At the end of each epoch, get the losses and print them out
train_loss_d = self.d_loss.eval({
self.inputs_x: batch_x,
self.inputs_z: batch_z
})
train_loss_g = self.g_loss.eval(
{self.inputs_z: batch_z})
print(
"Epoch {}/{}...".format(e + 1, self.epochs),
"Discriminator Loss: {:.4f}...".format(
train_loss_d),
"Generator Loss: {:.4f}".format(train_loss_g))
losses.append((train_loss_d, train_loss_g))
steps += 1
# save generation results at every epochs
if e % (self.d_train_freq * self.save_every) == 0:
val_z = np.random.uniform(-1,
1,
size=(val_size, self.z_dim))
val_out = sess.run(network.generator(self.inputs_z,
reuse=True,
is_training=False),
feed_dict={self.inputs_z: val_z})
image_fn = os.path.join(
self.assets_dir, '{:s}-val-e{:03d}.png'.format(
self.dataset_type, (e // self.d_train_freq + 1)))
utils.validation(val_out,
self.val_block_size,
image_fn,
color_mode='L')
end_time = time.time()
elapsed_time = end_time - start_time
# save losses as image
losses_fn = os.path.join(self.assets_dir,
'{:s}-losses.png'.format(self.dataset_type))
utils.save_losses(losses, ['Discriminator', 'Generator'], elapsed_time,
losses_fn)
return
argmax_indices = np.argmax(class_probabilities, axis=-1)
label_ids = b_labels.to('cpu').numpy()
seq_lengths = b_len.to('cpu').numpy()
for ix in range(len(label_ids)):
# Store predictions and true labels
pred_labels = [index2label[p] for p in argmax_indices[ix][:seq_lengths[ix]] if p != 0]
gold_labels = [index2label[g] for g in label_ids[ix] if g != 0]
eval_metrics = utils.evaluate_tagset(gold_labels, pred_labels, ignore_verb_label=False)
arg_excess, arg_missed, arg_match = utils._add_to_eval_dicts(eval_metrics, arg_excess, arg_missed, arg_match)
tot_excess += len(arg_excess)
tot_missed += len(arg_missed)
tot_match += len(arg_match)
# Report the final accuracy for this validation run.
logging.info(f"tp = {tot_match} || fp = {tot_excess} || fn = {tot_missed}")
p, r, f = utils.get_metrics(false_pos=tot_excess, false_neg=tot_missed, true_pos=tot_match)
logging.info(" Precision: {0:.2f} || Recall: {1:.2f} || F1: {2:.2f}".format(p, r, f))
logging.info(" Validation took: {:}".format(utils.format_time(time.time() - t0)))
# ================================================
# Save Checkpoint for this Epoch
# ================================================
utils.save_model(f"{MODEL_DIR}/EPOCH_{epoch_i}", {"args":[]}, model, tokenizer)
utils.save_losses(loss_values, filename=LOSS_FILENAME)
logging.info("")
logging.info("Training complete!")
def write_summary(train_loss, valid_loss, anomal_loss, start_thr, end_thr,
false_positives, true_positives, training_time,
best_features, io_dim):
'''
Saves losses curves, roc curves and model parameters
'''
# Init directory
summary_dir = './Summaries/'
model_dir = summary_dir + model_name + '/'
while os.path.exists(model_dir):
print('[ ! ] Model summary already exists.')
tmp_name = input('[ ? ] Choose new name for model directory: ')
model_dir = summary_dir + tmp_name + '/'
os.makedirs(model_dir)
# Save loss curves
loss_name = model_dir + 'losses.png'
utils.save_losses(train_loss, valid_loss, anomal_loss, loss_name)
# Save ROC curve
roc_name = model_dir + 'roc.png'
utils.save_roc(false_positives, true_positives, roc_name)
# Save parameters file
parameters_file = model_dir + 'parameters.txt'
parameters = open(parameters_file, 'w')
parameters.write('Document creation : {}\n'.format(time.strftime("%c")))
parameters.write('Model name : {}\n'.format(model_name))
parameters.write('Model type : seq2seq\n')
parameters.write('Dataset : {}\n'.format(dataset_path))
parameters.write('\nSequence length : {}\n'.format(seq_length))
parameters.write('Sequence dimension : {}'.format(io_dim))
parameters.write(
'Stacked layers count : {}\n'.format(layers_stacked_count))
parameters.write('Batch size : {}\n'.format(batch_size))
parameters.write('Epochs : {}\n'.format(epoch))
parameters.write('\nOptimizer : RMSprop\n')
parameters.write('Learning rate : {}\n'.format(learning_rate))
parameters.write('\nTraining time : {}\n'.format(
time.strftime('%H:%M:%S', time.localtime(training_time))))
parameters.write('\nROC settings :\n')
parameters.write('\tThreshold start : {}\n'.format(start_thr))
parameters.write('\tThreshold end : {}\n'.format(end_thr))
parameters.write('\tStep : {}\n'.format(step_thr))
if best_features is not None:
parameters.write('\nFeature selection : \n')
# Change std to print directly in file
old_std = sys.stdout
sys.stdout = parameters
# Draw table
table = BeautifulTable()
table.column_headers = ['Selected features']
for feature in best_features:
table.append_row([feature])
print(table)
# restore original stdout
sys.stdout = old_std
parameters.close()
print('[ + ] Summary saved !')