def train(self): # fix z for visualization n_fixed_samples = self.val_block_size * self.val_block_size fixed_z = np.random.uniform(-1.0, 1.0, size=(n_fixed_samples, self.z_dim)) # get dataset mnist_dataset = get_mnist_by_name(self.batch_size, self.dataset_type) # setup tracking variables steps = 0 losses = [] start_time = time.time() with tf.Session() as sess: # reset tensorflow variables sess.run(tf.global_variables_initializer()) # start training for e in range(self.epochs): # setup dataset iterator for graph mode iterator = mnist_dataset.make_one_shot_iterator() next_elem = iterator.get_next() while True: try: self.train_step(sess, next_elem, steps, losses) steps += 1 except tf.errors.OutOfRangeError: print('End of dataset') break # save generation results at every n epochs if e % self.save_every == 0: self.save_generator_output(sess, e, fixed_z) self.saver.save( sess, self.ckpt_fn, global_step=tf.train.get_or_create_global_step()) # save final output self.save_generator_output(sess, e, fixed_z) self.saver.save(sess, self.ckpt_fn, global_step=tf.train.get_or_create_global_step()) end_time = time.time() elapsed_time = end_time - start_time # save losses as image losses_fn = os.path.join( self.assets_dir, '{:s}-{:s}-losses.png'.format(self.dataset_type, self.gan_loss_type)) utils.save_losses(losses, ['Discriminator', 'Generator'], elapsed_time, losses_fn) return
def callback_fn(self, params): print("Function value: ", end='') loss = objective_function(params, self.X, self.y, self.lambd) print(loss) print("Average gradient: ", end='') avg_grad = np.mean( d_optimization_function(params, self.X, self.y, self.lambd)**2) print(avg_grad) print() self.iters += 1 save_params(params, self.filename, self.iters) save_losses(loss, self.loss_filename, self.iters)
def main(data_dir, plane, epochs, lr, weight_decay, device=None): diagnoses = ['abnormal', 'acl', 'meniscus'] exp = f'{datetime.now():%Y-%m-%d_%H-%M}' out_dir, losses_path = create_output_dir(exp, plane) if device is None: device = 'cuda' if torch.cuda.is_available() else 'cpu' print('Creating data loaders...') train_loader = make_data_loader(data_dir, 'train', plane, device, shuffle=True) valid_loader = make_data_loader(data_dir, 'valid', plane, device) print(f'Creating models...') # Create a model for each diagnosis models = [MRNet().to(device), MRNet().to(device), MRNet().to(device)] # Calculate loss weights based on the prevalences in train set pos_weights = calculate_weights(data_dir, 'train', device) criterions = [nn.BCEWithLogitsLoss(pos_weight=weight) \ for weight in pos_weights] optimizers = [make_adam_optimizer(model, lr, weight_decay) \ for model in models] lr_schedulers = [make_lr_scheduler(optimizer) for optimizer in optimizers] min_valid_losses = [np.inf, np.inf, np.inf] print(f'Training a model using {plane} series...') print(f'Checkpoints and losses will be save to {out_dir}') for epoch, _ in enumerate(range(epochs), 1): print(f'=== Epoch {epoch}/{epochs} ===') batch_train_losses = np.array([0.0, 0.0, 0.0]) batch_valid_losses = np.array([0.0, 0.0, 0.0]) for inputs, labels in train_loader: inputs, labels = inputs.to(device), labels.to(device) batch_loss = batch_forward_backprop(models, inputs, labels, criterions, optimizers) batch_train_losses += batch_loss valid_preds = [] valid_labels = [] for inputs, labels in valid_loader: inputs, labels = inputs.to(device), labels.to(device) batch_preds, batch_loss = \ batch_forward(models, inputs, labels, criterions) batch_valid_losses += batch_loss valid_labels.append(labels.detach().cpu().numpy().squeeze()) valid_preds.append(batch_preds) batch_train_losses /= len(train_loader) batch_valid_losses /= len(valid_loader) print_stats(batch_train_losses, batch_valid_losses, valid_labels, valid_preds) save_losses(batch_train_losses, batch_valid_losses, losses_path) update_lr_schedulers(lr_schedulers, batch_valid_losses) for i, (batch_v_loss, min_v_loss) in \ enumerate(zip(batch_valid_losses, min_valid_losses)): if batch_v_loss < min_v_loss: save_checkpoint(epoch, plane, diagnoses[i], models[i], optimizers[i], out_dir) min_valid_losses[i] = batch_v_loss
scheduler1.step() scheduler2.step() if confs['loss'] == 'wae-gan': scheduler1_disc.step() scheduler2_disc.step() train_loss, recon, match = train(model, trainloader, optimizer, confs, epoch) train_losses.append(train_loss) train_loss_data['total'].append(train_loss) train_loss_data['recon'].append(recon) train_loss_data['match'].append(match) if testloader != None: test_loss, recon, match = test(model, testloader, epoch, confs, add_noise=confs['noise']) test_losses.append(test_loss) test_loss_data['total'].append(train_loss) test_loss_data['recon'].append(recon) test_loss_data['match'].append(match) print('Epoch: {} Train Loss: {} Test Loss: {}'.format( epoch, train_loss, test_loss)) else: print('Epoch: {} Train Loss: {}'.format(epoch, train_loss)) if epoch % config.SAVEFREQ == 0: save_model(confs['dataset'] + '_' + confs['type'], model, epoch, confs) save_losses(train_loss_data, confs, epoch, 'train') save_losses(test_loss_data, confs, epoch, 'test') save_model(confs['dataset'], model, epoch, confs)
def train(self): n_fixed_samples = self.val_block_size * self.val_block_size fixed_z = np.random.uniform(-1, 1, size=(n_fixed_samples, self.z_dim)) fixed_y = np.zeros(shape=[n_fixed_samples, self.y_dim]) for s in range(n_fixed_samples): loc = s % self.y_dim fixed_y[s, loc] = 1 steps = 0 losses = [] start_time = time.time() with tf.Session() as sess: # reset tensorflow variables sess.run(tf.global_variables_initializer()) # start training for e in range(self.epochs): for ii in range(self.mnist_loader.train.num_examples // self.batch_size): batch_x, batch_y = self.mnist_loader.train.next_batch( self.batch_size) # rescale images to -1 ~ 1 batch_x = np.reshape(batch_x, (-1, 28, 28, 1)) batch_x = batch_x * 2.0 - 1.0 # Sample random noise for G batch_z = np.random.uniform(-1, 1, size=(self.batch_size, self.z_dim)) fd = { self.inputs_x: batch_x, self.inputs_y: batch_y, self.inputs_z: batch_z } # Run optimizers _ = sess.run(self.d_opt, feed_dict=fd) _ = sess.run(self.g_opt, feed_dict=fd) _ = sess.run(self.ac_opt, feed_dict=fd) # print losses if steps % self.print_every == 0: # At the end of each epoch, get the losses and print them out train_loss_d = self.d_loss.eval(fd) train_loss_g = self.g_loss.eval(fd) train_loss_ac = self.ac_loss.eval(fd) print( "Epoch {}/{}...".format(e + 1, self.epochs), "Discriminator Loss: {:.4f}...".format( train_loss_d), "Generator Loss: {:.4f}...".format(train_loss_g), "Auxilary Classifier Loss: {:.4f}...".format( train_loss_ac)) losses.append( (train_loss_d, train_loss_g, train_loss_ac)) steps += 1 # save generation results at every epochs if e % self.save_every == 0: val_out = sess.run(network.generator(self.inputs_z, y=self.inputs_y, reuse=True, is_training=False), feed_dict={ self.inputs_y: fixed_y, self.inputs_z: fixed_z }) image_fn = os.path.join( self.assets_dir, '{:s}-val-e{:03d}.png'.format(self.dataset_type, e + 1)) utils.validation(val_out, self.val_block_size, image_fn, color_mode='L') end_time = time.time() elapsed_time = end_time - start_time # save losses as image losses_fn = os.path.join(self.assets_dir, '{:s}-losses.png'.format(self.dataset_type)) utils.save_losses(losses, ['Discriminator', 'Generator', 'Auxilary'], elapsed_time, losses_fn) return
def train(self): val_size = self.val_block_size * self.val_block_size steps = 0 losses = [] new_epochs = self.d_train_freq * self.epochs start_time = time.time() with tf.Session() as sess: # reset tensorflow variables sess.run(tf.global_variables_initializer()) # start training for e in range(new_epochs): for ii in range(self.mnist_loader.train.num_examples // self.batch_size): # no need labels batch_x, _ = self.mnist_loader.train.next_batch( self.batch_size) # rescale images to -1 ~ 1 batch_x = np.reshape(batch_x, (-1, 28, 28, 1)) batch_x = batch_x * 2.0 - 1.0 # Sample random noise for G batch_z = np.random.uniform(-1, 1, size=(self.batch_size, self.z_dim)) fd = {self.inputs_x: batch_x, self.inputs_z: batch_z} # Run optimizers (train D more than G) _ = sess.run(self.d_weight_clip) _ = sess.run(self.d_opt, feed_dict=fd) if ii % self.d_train_freq == 0: _ = sess.run(self.g_opt, feed_dict=fd) # print losses if steps % self.print_every == 0: # At the end of each epoch, get the losses and print them out train_loss_d = self.d_loss.eval({ self.inputs_x: batch_x, self.inputs_z: batch_z }) train_loss_g = self.g_loss.eval( {self.inputs_z: batch_z}) print( "Epoch {}/{}...".format(e + 1, self.epochs), "Discriminator Loss: {:.4f}...".format( train_loss_d), "Generator Loss: {:.4f}".format(train_loss_g)) losses.append((train_loss_d, train_loss_g)) steps += 1 # save generation results at every epochs if e % (self.d_train_freq * self.save_every) == 0: val_z = np.random.uniform(-1, 1, size=(val_size, self.z_dim)) val_out = sess.run(network.generator(self.inputs_z, reuse=True, is_training=False), feed_dict={self.inputs_z: val_z}) image_fn = os.path.join( self.assets_dir, '{:s}-val-e{:03d}.png'.format( self.dataset_type, (e // self.d_train_freq + 1))) utils.validation(val_out, self.val_block_size, image_fn, color_mode='L') end_time = time.time() elapsed_time = end_time - start_time # save losses as image losses_fn = os.path.join(self.assets_dir, '{:s}-losses.png'.format(self.dataset_type)) utils.save_losses(losses, ['Discriminator', 'Generator'], elapsed_time, losses_fn) return
argmax_indices = np.argmax(class_probabilities, axis=-1) label_ids = b_labels.to('cpu').numpy() seq_lengths = b_len.to('cpu').numpy() for ix in range(len(label_ids)): # Store predictions and true labels pred_labels = [index2label[p] for p in argmax_indices[ix][:seq_lengths[ix]] if p != 0] gold_labels = [index2label[g] for g in label_ids[ix] if g != 0] eval_metrics = utils.evaluate_tagset(gold_labels, pred_labels, ignore_verb_label=False) arg_excess, arg_missed, arg_match = utils._add_to_eval_dicts(eval_metrics, arg_excess, arg_missed, arg_match) tot_excess += len(arg_excess) tot_missed += len(arg_missed) tot_match += len(arg_match) # Report the final accuracy for this validation run. logging.info(f"tp = {tot_match} || fp = {tot_excess} || fn = {tot_missed}") p, r, f = utils.get_metrics(false_pos=tot_excess, false_neg=tot_missed, true_pos=tot_match) logging.info(" Precision: {0:.2f} || Recall: {1:.2f} || F1: {2:.2f}".format(p, r, f)) logging.info(" Validation took: {:}".format(utils.format_time(time.time() - t0))) # ================================================ # Save Checkpoint for this Epoch # ================================================ utils.save_model(f"{MODEL_DIR}/EPOCH_{epoch_i}", {"args":[]}, model, tokenizer) utils.save_losses(loss_values, filename=LOSS_FILENAME) logging.info("") logging.info("Training complete!")
def write_summary(train_loss, valid_loss, anomal_loss, start_thr, end_thr, false_positives, true_positives, training_time, best_features, io_dim): ''' Saves losses curves, roc curves and model parameters ''' # Init directory summary_dir = './Summaries/' model_dir = summary_dir + model_name + '/' while os.path.exists(model_dir): print('[ ! ] Model summary already exists.') tmp_name = input('[ ? ] Choose new name for model directory: ') model_dir = summary_dir + tmp_name + '/' os.makedirs(model_dir) # Save loss curves loss_name = model_dir + 'losses.png' utils.save_losses(train_loss, valid_loss, anomal_loss, loss_name) # Save ROC curve roc_name = model_dir + 'roc.png' utils.save_roc(false_positives, true_positives, roc_name) # Save parameters file parameters_file = model_dir + 'parameters.txt' parameters = open(parameters_file, 'w') parameters.write('Document creation : {}\n'.format(time.strftime("%c"))) parameters.write('Model name : {}\n'.format(model_name)) parameters.write('Model type : seq2seq\n') parameters.write('Dataset : {}\n'.format(dataset_path)) parameters.write('\nSequence length : {}\n'.format(seq_length)) parameters.write('Sequence dimension : {}'.format(io_dim)) parameters.write( 'Stacked layers count : {}\n'.format(layers_stacked_count)) parameters.write('Batch size : {}\n'.format(batch_size)) parameters.write('Epochs : {}\n'.format(epoch)) parameters.write('\nOptimizer : RMSprop\n') parameters.write('Learning rate : {}\n'.format(learning_rate)) parameters.write('\nTraining time : {}\n'.format( time.strftime('%H:%M:%S', time.localtime(training_time)))) parameters.write('\nROC settings :\n') parameters.write('\tThreshold start : {}\n'.format(start_thr)) parameters.write('\tThreshold end : {}\n'.format(end_thr)) parameters.write('\tStep : {}\n'.format(step_thr)) if best_features is not None: parameters.write('\nFeature selection : \n') # Change std to print directly in file old_std = sys.stdout sys.stdout = parameters # Draw table table = BeautifulTable() table.column_headers = ['Selected features'] for feature in best_features: table.append_row([feature]) print(table) # restore original stdout sys.stdout = old_std parameters.close() print('[ + ] Summary saved !')