def transfer(model, decoder, sess, args, vocab, data0, data1, out_path): batches, order0, order1 = get_batches(data0, data1, vocab.word2id, args.batch_size) data0_tsf, data1_tsf = [], [] losses = Losses(len(batches)) for batch in batches: ori, tsf = decoder.rewrite(batch) half = batch['size'] / 2 data0_tsf += tsf[:half] data1_tsf += tsf[half:] loss, loss_g, loss_d, loss_d0, loss_d1 = sess.run( [ model.loss, model.loss_g, model.loss_d, model.loss_d0, model.loss_d1 ], feed_dict=feed_dictionary(model, batch, args.rho, args.gamma_min)) losses.add(loss, loss_g, loss_d, loss_d0, loss_d1) n0, n1 = len(data0), len(data1) data0_tsf = reorder(order0, data0_tsf)[:n0] data1_tsf = reorder(order1, data1_tsf)[:n1] if out_path: write_sent(data0_tsf, out_path + '.0' + '.tsf') write_sent(data1_tsf, out_path + '.1' + '.tsf') return losses
def __init__(self, model, optimizer, train_loader, test_loader, args, epoch=-1, global_step=0, test_mode=False): if args.fp16: try: from apex import amp global amp amp = amp except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use fp16 training." ) self.model = model self.args = args self.optimizer = optimizer self.train_loader = train_loader self.test_loader = test_loader self.epoch = epoch self.module = model.module if hasattr( model, 'module') else model # for data parallel self.masking_policies = [ 'random', 'seen_noun', 'seen_verb', 'seen_combo_seen_noun_seen_verb', 'new_noun', 'new_verb', 'new_combo_seen_noun_seen_verb', 'new_combo_new_noun_new_verb', 'seen_combo_seen_noun_seen_verb_merge', 'new_combo_seen_noun_seen_verb_merge', 'new_combo_new_noun_new_verb_merge' ] if test_mode and not args.pointing: self.masker = TestMasker(annotation_root=args.annotation_root, masking_policy=args.test_masking_policy, tok=self.train_loader.dataset.tokenizer, p_mask_img=args.p_mask_img, p_mask_txt=args.p_mask_txt) else: self.masker = Masker(self.train_loader.dataset.tokenizer, **vars(args)) self.losses = Losses(self.module.cfg, args, **vars(args)) self.global_step = global_step
def transfer(model, sess, args, vocab, data0, data1, out_path): batches, order0, order1 = get_batches(data0, data1, vocab.word2id, args.batch_size) data0_tsf, data1_tsf = [], [] losses = Losses(len(batches)) for batch in batches: ori, tsf, loss, loss_g, loss_d, loss_d0, loss_d1 = rewrite( model, sess, args, vocab, batch) half = batch['size'] / 2 data0_tsf += tsf[:half] data1_tsf += tsf[half:] losses.add(loss, loss_g, loss_d, loss_d0, loss_d1) n0, n1 = len(data0), len(data1) data0_tsf = reorder(order0, data0_tsf)[:n0] data1_tsf = reorder(order1, data1_tsf)[:n1] if out_path: write_sent(data0_tsf, out_path + '.0' + '.tsf') write_sent(data1_tsf, out_path + '.1' + '.tsf') return losses
with tf.Session(config=config) as sess: model = create_model(sess, args, vocab) if args.beam > 1: decoder = beam_search.Decoder(sess, args, vocab, model) else: decoder = greedy_decoding.Decoder(sess, args, vocab, model) if args.train: batches, _, _ = get_batches(train0, train1, vocab.word2id, args.batch_size) random.shuffle(batches) start_time = time.time() step = 0 losses = Losses(args.steps_per_checkpoint) best_dev = float('inf') learning_rate = args.learning_rate rho = args.rho gamma = args.gamma_init dropout = args.dropout_keep_prob for epoch in range(1, 1 + args.max_epochs): print '--------------------epoch %d--------------------' % epoch print 'learning_rate:', learning_rate, ' gamma:', gamma for batch in batches: feed_dict = feed_dictionary(model, batch, rho, gamma, dropout, learning_rate) loss_d0, _ = sess.run([model.loss_d0, model.optimizer_d0],
def train(**args): """ Evaluate selected model Args: rerun (Int): Integer indicating number of repetitions for the select experiment seed (Int): Integer indicating set seed for random state save_dir (String): Top level directory to generate results folder model (String): Name of selected model dataset (String): Name of selected dataset exp (String): Name of experiment debug (Int): Debug state to avoid saving variables load_type (String): Keyword indicator to evaluate the testing or validation set pretrained (Int/String): Int/String indicating loading of random, pretrained or saved weights opt (String): Int/String indicating loading of random, pretrained or saved weights lr (Float): Learning rate momentum (Float): Momentum in optimizer weight_decay (Float): Weight_decay value final_shape ([Int, Int]): Shape of data when passed into network Return: None """ print( "\n############################################################################\n" ) print("Experimental Setup: ", args) print( "\n############################################################################\n" ) for total_iteration in range(args['rerun']): # Generate Results Directory d = datetime.datetime.today() date = d.strftime('%Y%m%d-%H%M%S') result_dir = os.path.join( args['save_dir'], args['model'], '_'.join( (args['dataset'], args['exp'], date))) log_dir = os.path.join(result_dir, 'logs') save_dir = os.path.join(result_dir, 'checkpoints') if not args['debug']: os.makedirs(result_dir, exist_ok=True) os.makedirs(log_dir, exist_ok=True) os.makedirs(save_dir, exist_ok=True) # Save copy of config file with open(os.path.join(result_dir, 'config.yaml'), 'w') as outfile: yaml.dump(args, outfile, default_flow_style=False) # Tensorboard Element writer = SummaryWriter(log_dir) # Check if GPU is available (CUDA) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # Load Network model = create_model_object(**args).to(device) # Load Data loader = data_loader(model_obj=model, **args) if args['load_type'] == 'train': train_loader = loader['train'] valid_loader = loader[ 'train'] # Run accuracy on train data if only `train` selected elif args['load_type'] == 'train_val': train_loader = loader['train'] valid_loader = loader['valid'] else: sys.exit('Invalid environment selection for training, exiting') # END IF # Training Setup params = [p for p in model.parameters() if p.requires_grad] if args['opt'] == 'sgd': optimizer = optim.SGD(params, lr=args['lr'], momentum=args['momentum'], weight_decay=args['weight_decay']) elif args['opt'] == 'adam': optimizer = optim.Adam(params, lr=args['lr'], weight_decay=args['weight_decay']) else: sys.exit('Unsupported optimizer selected. Exiting') # END IF scheduler = MultiStepLR(optimizer, milestones=args['milestones'], gamma=args['gamma']) if isinstance(args['pretrained'], str): ckpt = load_checkpoint(args['pretrained']) model.load_state_dict(ckpt) start_epoch = load_checkpoint(args['pretrained'], key_name='epoch') + 1 optimizer.load_state_dict( load_checkpoint(args['pretrained'], key_name='optimizer')) for quick_looper in range(start_epoch): scheduler.step() # END FOR else: start_epoch = 0 # END IF model_loss = Losses(device=device, **args) acc_metric = Metrics(**args) best_val_acc = 0.0 ############################################################################################################################################################################ # Start: Training Loop for epoch in range(start_epoch, args['epoch']): running_loss = 0.0 print('Epoch: ', epoch) # Setup Model To Train model.train() # Start: Epoch for step, data in enumerate(train_loader): if step % args['pseudo_batch_loop'] == 0: loss = 0.0 optimizer.zero_grad() # END IF x_input = data['data'].to(device) annotations = data['annots'] assert args['final_shape'] == list(x_input.size( )[-2:]), "Input to model does not match final_shape argument" outputs = model(x_input) loss = model_loss.loss(outputs, annotations) loss = loss * args['batch_size'] loss.backward() running_loss += loss.item() if np.isnan(running_loss): import pdb pdb.set_trace() # END IF if not args['debug']: # Add Learning Rate Element for param_group in optimizer.param_groups: writer.add_scalar( args['dataset'] + '/' + args['model'] + '/learning_rate', param_group['lr'], epoch * len(train_loader) + step) # END FOR # Add Loss Element writer.add_scalar( args['dataset'] + '/' + args['model'] + '/minibatch_loss', loss.item() / args['batch_size'], epoch * len(train_loader) + step) # END IF if ((epoch * len(train_loader) + step + 1) % 100 == 0): print('Epoch: {}/{}, step: {}/{} | train loss: {:.4f}'. format( epoch, args['epoch'], step + 1, len(train_loader), running_loss / float(step + 1) / args['batch_size'])) # END IF if (epoch * len(train_loader) + (step + 1)) % args['pseudo_batch_loop'] == 0 and step > 0: # Apply large mini-batch normalization for param in model.parameters(): param.grad *= 1. / float( args['pseudo_batch_loop'] * args['batch_size']) optimizer.step() # END IF # END FOR: Epoch if not args['debug']: # Save Current Model save_path = os.path.join( save_dir, args['dataset'] + '_epoch' + str(epoch) + '.pkl') save_checkpoint(epoch, step, model, optimizer, save_path) # END IF: Debug scheduler.step(epoch=epoch) print('Schedulers lr: %f', scheduler.get_lr()[0]) ## START FOR: Validation Accuracy running_acc = [] running_acc = valid(valid_loader, running_acc, model, device, acc_metric) if not args['debug']: writer.add_scalar( args['dataset'] + '/' + args['model'] + '/validation_accuracy', 100. * running_acc[-1], epoch * len(valid_loader) + step) print('Accuracy of the network on the validation set: %f %%\n' % (100. * running_acc[-1])) # Save Best Validation Accuracy Model Separately if best_val_acc < running_acc[-1]: best_val_acc = running_acc[-1] if not args['debug']: # Save Current Model save_path = os.path.join( save_dir, args['dataset'] + '_best_model.pkl') save_checkpoint(epoch, step, model, optimizer, save_path) # END IF # END IF # END FOR: Training Loop ############################################################################################################################################################################ if not args['debug']: # Close Tensorboard Element writer.close()
def __init__( self, seed=None, optimizer=Adam, optimizer_kwargs={}, learning_rate_init=0.04, gamma=0.995, # learning rate decay factor considered_groups=list( range(12)), ## group layers to be considered from start sample_variance_threshold=0.002, weight_loss_sample_variance=0, # 10. evaluation_steps=250, # number of batches between loss tracking N_batches_test=1, # number of batches considered for evaluation ): super(ImageClassifier, self).__init__(considered_groups=considered_groups) if seed is not None: torch.manual_seed(seed) #''' resnet = models.resnet18(pretrained=False) self.conv = Sequential( *(list(resnet.children())[:-1]), Flatten(), ) ''' architecture used by Dielemann et al 2015 self.conv = Sequential( # Conv2dUntiedBias(41, 41, 3, 32, kernel_size=6), Conv2d(3,32, kernel_size=6), ReLU(), MaxPool2d(2), # Conv2dUntiedBias(16, 16, 32, 64, kernel_size=5), Conv2d(32, 64, kernel_size=5), ReLU(), MaxPool2d(2), # Conv2dUntiedBias(6, 6, 64, 128, kernel_size=3), Conv2d(64, 128, kernel_size=3), ReLU(), # Conv2dUntiedBias(4, 4, 128, 128, kernel_size=3), #weight_std=0.1), Conv2d(128, 128, kernel_size=3), ReLU(), MaxPool2d(2), Flatten(), ) #''' self.dense1 = MaxOut(8192, 2048, bias=0.01) self.dense2 = MaxOut(2048, 2048, bias=0.01) self.dense3 = Sequential( MaxOut(2048, 37, bias=0.1), # LeakyReLU(negative_slope=1e-7), ALReLU(negative_slope=1e-2), ) self.dropout = Dropout(p=0.5) self.augment = Compose([ Lambda(lambda img: torch.cat([img, hflip(img)], 0)), Lambda(lambda img: torch.cat([img, rotate(img, 45)], 0)), FiveCrop(45), Lambda(lambda crops: torch.cat([ rotate(crop, ang) for crop, ang in zip(crops, (0, 90, 270, 180)) ], 0)), ]) self.N_augmentations = 16 self.N_conv_outputs = 512 self.set_optimizer(optimizer, lr=learning_rate_init, **optimizer_kwargs) # self.scheduler = ExponentialLR(self.optimizer, gamma=gamma) self.scheduler = MultiStepLR(self.optimizer, milestones=[292, 373], gamma=gamma) self.make_labels_hierarchical = False # if True, output probabilities are renormalized to fit the hierarchical label structure self.N_batches_test = N_batches_test self.evaluation_steps = evaluation_steps # number of batches between loss tracking self.weight_loss_sample_variance = weight_loss_sample_variance self.sample_variance_threshold = sample_variance_threshold self.iteration = 0 self.epoch = 0 self.losses_train = Losses("loss", "train") self.losses_valid = Losses("loss", "valid") self.sample_variances_train = Losses("sample variance", "train") self.sample_variances_valid = Losses("sample variance", "valid") for g in range(1, 12): setattr(self, f"accuracies_Q{g}_train", Accuracies("accuracy train", f"Q{g}")) setattr(self, f"accuracies_Q{g}_valid", Accuracies("accuracy valid", f"Q{g}")) self.losses_regression = Losses("loss", "regression") self.losses_variance = Losses("loss", "sample variance") ## return to random seed if seed is not None: sd = np.random.random() * 10000 torch.manual_seed(sd)
def unet_model_fn(features, labels, mode, params): tf.local_variables_initializer() loss, train_op, = None, None eval_metric_ops, training_hooks, evaluation_hooks = None, None, None predictions_dict = None unet = Unet(params=params) logits = unet.model(input_tensor=features['image']) y_pred = tf.math.softmax(logits, axis=-1) output_img = tf.expand_dims(tf.cast(tf.math.argmax(y_pred, axis=-1) * 255, dtype=tf.uint8), axis=-1) if mode in (estimator.ModeKeys.TRAIN, estimator.ModeKeys.EVAL): with tf.name_scope('Loss_Calculation'): loss = Losses(logits=logits, labels=labels['label']) loss = loss.custom_loss() with tf.name_scope('Dice_Score_Calculation'): dice = f1(labels=labels['label'], predictions=y_pred) with tf.name_scope('Images_{}'.format(mode)): with tf.name_scope('Reformat_Outputs'): label = tf.expand_dims(tf.cast(tf.argmax(labels['label'], -1) * 255, dtype=tf.uint8), axis=-1) image = tf.math.divide(features['image'] - tf.reduce_max(features['image'], [0, 1, 2]), tf.reduce_max(features['image'], [0, 1, 2]) - tf.reduce_min(features['image'], [0, 1, 2])) summary.image('1_Medical_Image', image, max_outputs=1) summary.image('2_Output', output_img, max_outputs=1) summary.image('3_Output_pred', tf.expand_dims(y_pred[:, :, :, 1], -1), max_outputs=1) summary.image('4_Output_label', label, max_outputs=1) if mode == estimator.ModeKeys.TRAIN: with tf.name_scope('Learning_Rate'): global_step = tf.compat.v1.train.get_or_create_global_step() learning_rate = tf.compat.v1.train.exponential_decay(params['lr'], global_step=global_step, decay_steps=params['decay_steps'], decay_rate=params['decay_rate'], staircase=False) with tf.name_scope('Optimizer_conf'): train_op = Adam(learning_rate=learning_rate).minimize(loss=loss, global_step=global_step) with tf.name_scope('Metrics'): summary.scalar('Output_DSC', dice[1]) summary.scalar('Learning_Rate', learning_rate) if mode == estimator.ModeKeys.EVAL: eval_metric_ops = {'Metrics/Output_DSC': dice} eval_summary_hook = tf.estimator.SummarySaverHook(output_dir=params['eval_path'], summary_op=summary.merge_all(), save_steps=params['eval_steps']) evaluation_hooks = [eval_summary_hook] if mode == estimator.ModeKeys.PREDICT: predictions_dict = {'image': features['image'], 'y_preds': y_pred[:, :, :, 1], 'output_img': output_img, 'path': features['path']} return estimator.EstimatorSpec(mode, predictions=predictions_dict, loss=loss, train_op=train_op, eval_metric_ops=eval_metric_ops, training_hooks=training_hooks, evaluation_hooks=evaluation_hooks)