def test_loss_wrapper_affinity_masking(self): from neurofire.criteria.loss_transforms import MaskTransitionToIgnoreLabel from neurofire.criteria.loss_transforms import RemoveSegmentationFromTarget from neurofire.criteria.loss_wrapper import LossWrapper from neurofire.transform.affinities import Segmentation2Affinities offsets = [(1, 0, 0), (0, 1, 0), (0, 0, 1), (9, 0, 0), (0, 9, 0), (0, 0, 9), (9, 4, 0), (4, 9, 0), (9, 0, 9)] trafos = Compose(MaskTransitionToIgnoreLabel(offsets, ignore_label=0), RemoveSegmentationFromTarget()) aff_trafo = Segmentation2Affinities(offsets, retain_segmentation=True) seg = self.make_segmentation_with_ignore(self.shape) ignore_mask = self.brute_force_transition_masking(seg, offsets) target = Variable(torch.Tensor(aff_trafo(seg.astype('float32'))[None]), requires_grad=False) tshape = target.size() pshape = (tshape[0], tshape[1] - 1) + tshape[2:] prediction = Variable(torch.Tensor(*pshape).uniform_(0, 1), requires_grad=True) # apply cross entropy loss criterion = BCELoss() # criterion = SorensenDiceLoss() wrapper = LossWrapper(criterion, trafos) loss = wrapper.forward(prediction, target) loss.backward() grads = prediction.grad.data.numpy().squeeze() self.assertEqual(grads.shape, ignore_mask.shape) self.assertTrue((grads[ignore_mask] == 0).all()) self.assertFalse(np.sum(grads[np.logical_not(ignore_mask)]) == 0)
def standard_cross_entropy(log_p_pred, log_r_pred, t_pred, y_true, r_true, t_true, log_r_clip=25.): r_pred = torch.exp(log_r_pred) # r_pred = torch.exp(torch.clamp(log_r_pred, -log_r_clip, log_r_clip)) s_hat = 1. / (1. + r_pred) return BCELoss()(s_hat, y_true)
def fit(self, points, features, actions, n_epoch=50000, lr=5e-4, batch_size=150): datasets = [] optimizer = optim.Adam(params=self.net.parameters(), lr=lr) bceLoss = BCELoss() for p, f, a in zip(points, features, actions): one_set = TensorDataset(to_tensor(p[:-1]), to_tensor(f[:-1]), to_tensor(a)) datasets.append(one_set) train_loader = DataLoader(ConcatDataset(datasets[:-1]), batch_size=1, shuffle=True) test_loader = DataLoader(ConcatDataset(datasets[-1:]), batch_size=1, shuffle=False) print(len(train_loader), len(test_loader)) for e in range(n_epoch): print(f"Epoch: {e}") for i, (p, f, a) in enumerate(train_loader): out = self.net(p, f) loss = bceLoss(out, a) loss.backward() if i % batch_size == 0: optimizer.step() optimizer.zero_grad() sum_loss = 0 with torch.no_grad(): for i, (p, f, a) in enumerate(train_loader): out = self.net(p, f) loss = bceLoss(out, a) sum_loss += loss sum_loss /= len(train_loader) print(f"Train Loss: {sum_loss.item()}") sum_loss = 0 with torch.no_grad(): for i, (p, f, a) in enumerate(test_loader): out = self.net(p, f) loss = bceLoss(out, a) sum_loss += loss sum_loss /= len(test_loader) print(f"Test Loss: {sum_loss.item()}")
def __init__(self, baseloss, classifier, lambda_clfloss=0.5, increase_until=10000, *args): super(FlipLoss, self).__init__(*args) self.baseloss = baseloss # assumed to return logit for binary classification (sigmoid) self.classifier = classifier self.sigmoid = nn.Sigmoid() self.bce = BCELoss() self.lambda_clfloss = lambda_clfloss self.increase_until = increase_until self.i = 0 for p in self.classifier.parameters(): p.requires_grad = False self.classifier.eval()
def create_setting_pretraindcae(model=None): """ This part is fixed for pretrain DCAE for mnist from paper Deep one-class classification setting. adam are used in paper. """ setting = {} if cfg.pretrain_solver == 'adam': optimizer = torch.optim.Adam(params=model.parameters(), lr=cfg.pretrain_lr) elif cfg.pretrain_solver == 'sgd': optimizer = torch.optim.SGD(lr=cfg.pretrain_lr, momentum=cfg.pretrain_momentum, nesterov=True, params=model.parameters()) else: raise ValueError('invalid pretrain solver for using: {}'.format(cfg.pretrain_solver)) setting['optim'] = optimizer if cfg.ae_loss == 'l2': #from loss import MSEReconstructionError #loss = MSEReconstructionError() print('using MSE') loss = MSELoss(reduction='none') if cfg.ae_loss == 'ce': loss = BCELoss() setting['criterion'] = loss return setting
# model ==================================================================== print("Init model...") # init model with pre-trained weights model = create_model() model.load_state_dict(torch.load('../trained_models/baseline_weights/gen_model.pt')) model.train() model.cuda() cudnn.benchmark = True # loss ===================================================================== print("BCE criterium...") bce_loss = BCELoss() # select only decoder parameters, keep vgg16 with pretrained weights decoder_parameters = [] for i, (a, p) in enumerate(model.named_parameters()): if i>25: print(i, a, p.shape) decoder_parameters.append(p) # optimizer parameters, added nesterov momentum optimizer = SGD(decoder_parameters, lr = args.lr, momentum=0.9, weight_decay=0.00001, nesterov=True)
opt = opts.parse_opt() if "CUDA_VISIBLE_DEVICES" not in os.environ.keys(): os.environ["CUDA_VISIBLE_DEVICES"] =opt.gpu #opt.model ='lstm' #opt.model ='fasttext' train_iter, test_iter = utils.loadData(opt) model=models.setup(opt) if torch.cuda.is_available(): model.cuda() model.train() print("# parameters:", sum(param.numel() for param in model.parameters())) optimizer = optim.Adam(model.parameters(), lr=opt.learning_rate) optimizer.zero_grad() loss_fun = BCELoss() #batch = next(iter(train_iter)) #x=batch.text[0] #print(utils.evaluation(model,test_iter)) for i in range(opt.max_epoch): for epoch,batch in enumerate(train_iter): start= time.time() predicted = model(batch.text[0]) loss= F.cross_entropy(predicted,batch.label) loss.backward() utils.clip_gradient(optimizer, opt.grad_clip) optimizer.step() if epoch% 100==0: if torch.cuda.is_available():
dataset = DiabetesDataset('diabetes.csv') train_loader = DataLoader(dataset=dataset, batch_size=32, shuffle=True, num_workers=2) model = Model() optim = SGD(model.parameters(), lr=0.1) num_epochs = 5 for epoch in range(num_epochs): for i, data in enumerate(train_loader, 0): inputs, labels = data inputs.to(device) labels.to(device) y_pred = model(inputs) y_pred = y_pred.squeeze(dim=1) print("Y_pred", y_pred) print("Y_true", labels) loss = BCELoss(y_pred, labels) print(epoch, i, loss.item()) optim.zero_grad() loss.backward() optim.step()
# Init optimizers print('Initializing the optimizers...') g_adam = Adam(g.parameters(), lr=gen_lr, weight_decay=dis_l2_reg) d_adam = Adam(d.parameters(), lr=dis_lr, weight_decay=gen_l2_reg) # Load the dataset print("Loading dataset...") transform = Compose([Resize((IMAGE_HEIGHT, IMAGE_WIDTH)), ToTensor()]) char_ds = CharacterDataset(dataset_path, labels_file, transform) loader = DataLoader(char_ds, batch_size=batch_size, shuffle=True) # Restore the content of the dataset if the training is not new if args.add: next_letter_to_add = args.add last_char_added_index = character_to_index_mapping[args.add] for curr_char in character_to_index_mapping.keys(): if curr_char == ' ': # the space is already present pass if curr_char == next_letter_to_add: break else: char_ds.add_character_to_training(curr_char) # Train print("Initiating GAN...") gan = CGAN(g, d, BCELoss(), BCELoss(), G_optim=g_adam, D_optim=d_adam, dataset_loader=loader, dataset=char_ds, device=dev, writer=writer, current_datetime=current_datetime) gan.train(add_character_every*len(character_to_index_mapping) + 1000, next_letter_to_add=next_letter_to_add, use_soft_labels=args.soft_labels)
if (target_numpy[sorted_output_indexes[i]] == 1.0): matches += 1 i += 1 accuracy = matches / match_by return accuracy model = Model() model.setup(80, 80, 6400, 3) hotspot = HotSpotDataset('train_1.npz', 'train') #i = 100 ; # id 2277411 for i in range(hotspot.__len__()): ids, inputs, targets = hotspot.__getitem__(i) x = Variable(inputs, requires_grad=True).float() y = Variable(targets, requires_grad=False).float() criterion = BCELoss() loss = criterion(model.forward(x), y) #criterion = BCEWithLogitsLoss(); loss = criterion(model.forward(x), y) #criterion = MultiMarginLoss(); loss = criterion(model.forward(x), y.long()) #criterion = MultiLabelMarginLoss(); loss = criterion(model.forward(x), y.long()) #criterion = MultiLabelSoftMarginLoss(); loss = criterion(model.forward(x), y) #loss = binary_cross_entropy(model.forward(x), y); #loss = binary_cross_entropy_with_logits(x, y); #x=CustomLoss()(x) #CustomLoss.apply # loss = nn.MSELoss() #loss=CustomLoss.apply() #loss=model.loss(x, y, 10); #loss = criterion(model.forward(x), y)
classifier = ClassifierFromElmo(1024, attn = 1) data = torch.randn(MAX_LEN, 200, 1024).to(device) mask = torch.ones(MAX_LEN, 200, 2*1024).to(device) yy = classifier(data, mask) print(yy.size()) classifier.save('classifiertry') newc = ClassifierFromElmo(1024, attn = 1, filen = 'classifiertry') zz = classifier(data, mask) ff = (yy-zz).sum() assert ff.abs() < 0.000001 print('works') assert False cl = time.clock() trload, teload = create_embed_loaders() print('done in {}'.format(time.clock()-cl)) classifier = ClassifierFromElmo(1024).cuda() for i in range(100): for j, b in enumerate(trload): x, y, mask = b['x'], b['y'], b['mask'] mask = mask.squeeze(1) classifier.zero_grad() yhat = classifier(x, mask) loss = BCE(yhat, y) loss.backward() print(loss.item())
def __init__(self): super().__init__() self.bce = BCELoss(reduction="sum") self.kl = lambda mu, logvar : torch.sum(-logvar + torch.exp(logvar) + mu ** 2 - 1) / 2
def __init__(self, beta=0.1): super().__init__() self.bce = BCELoss(reduction="sum") self.beta = beta self.mse = MSELoss(reduction="sum")
def forward(self, x): x = self.dense1(x) x = self.dense2(x) x = F.sigmoid(self.dense3(x)) return x if __name__ == '__main__': sparkConf = init_spark_conf().setAppName("testNNClassifer").setMaster( 'local[1]') sc = init_nncontext(sparkConf) sqlContext = SQLContext(sc) df = get_df(sqlContext) torch_model = SimpleTorchModel() becloss = BCELoss() model = TorchNet.from_pytorch(module=torch_model, input_shape=[1, 2], lossFunc=becloss.forward, pred_shape=[1, 1], label_shape=[1, 1]) classifier = NNEstimator(model, TorchIdentityCriterion(), SeqToTensor([2])) \ .setBatchSize(2) \ .setOptimMethod(Adam()) \ .setLearningRate(0.1) \ .setMaxEpoch(20) nnClassifierModel = classifier.fit(df) print("After training: ")
def __init__(self, name=None, initial_learning_rate=0.001, max_epochs=1000, batch_size=2, improvement_patience=20, optimizer_class=Adam, loss_function=BCELoss(), best_model_by_fscore=False, n_epochs_per_checkpoint=10, checkpoint_export_file=None, lr_refinement_multiplier=0.2, number_of_refinement_steps=10, refinement_patience=2, best_params_file="default_model.tsd", ): """Initialize a training strategy. Includes some validation params. :param name: Name the training strategy. This is useful for keeping track of log files: logging will use this name, e.g. as a TensorBoard comment. :param initial_learning_rate: Initial learning rate. Passed to the optimizer :param max_epochs: :param batch_size: :param improvement_patience: How many epochs are we willing to train without seeing improvement on the validation data before early-stopping? :param optimizer_class: A PyTorch optimizer class, like Adam. (The *class*, not an *instance* of the class.) :param loss_function: A PyTorch loss, like BCEWithLogitsLoss. :param best_model_by_fscore: Use the validation aggregated f-score instead of the loss to keep the best model during training. :param checkpoint_export_file: Where to dump the checkpoint params? :param n_epochs_per_checkpoint: Dump checkpoint params export once per this many epochs. :param number_of_refinement_steps: How many times should we try to attenuate the learning rate. :param lr_refinement_multiplier: Attenuate learning rate by this multiplicative factor in each refinement step. :param refinement_patience: How many epochs do we wait for improvement when in the refining stage. :param best_params_file: The file to which to save the best model. """ self.name = name # Learning rate self.initial_learning_rate = initial_learning_rate # Epochs & batches self.max_epochs = max_epochs self.batch_size = batch_size self.improvement_patience = improvement_patience # Loss function self.loss_function = loss_function # Optimizer self.optimizer_class = optimizer_class # Model selection self.best_model_by_fscore = best_model_by_fscore # Checkpointing self.n_epochs_per_checkpoint = n_epochs_per_checkpoint self.checkpoint_export_file = checkpoint_export_file # Early-stopping & Refinement self.lr_refinement_multiplier = lr_refinement_multiplier self.number_of_refinement_steps = number_of_refinement_steps self.refinement_patience = refinement_patience # Persisting the model self.best_params_file = best_params_file
def train(train_loader, dev_loader, model, label2id): add_classifier(model=model, labels=label2id) id2label = {v: k for k, v in label2id.items()} if torch.cuda.is_available(): model.to('cuda') max_epochs = model.configuration('MaxEpoch') lr = model.configuration('LearningRate') max_patience = model.configuration('Patience') epoch = 0 early_stop = False criterion = BCELoss() optimizer = Adam(lr=lr, params=model.parameters()) while epoch < max_epochs and not early_stop: model.train() running_loss = 0.0 steps = 0 bar = tqdm(total=len(train_loader), desc='Training') for batch in train_loader: batched_sentences, batched_attn, batched_labels = batch if torch.cuda.is_available(): batched_sentences = batched_sentences.cuda() batched_attn = batched_attn.cuda() batched_labels = batched_labels.cuda() outputs = model(batched_sentences, attention_mask=batched_attn) optimizer.zero_grad() loss = criterion(outputs, batched_labels) loss.backward() optimizer.step() running_loss += loss.item() steps += 1 bar.update(1) bar.close() loss = running_loss / steps with torch.no_grad(): bar = tqdm(total=len(dev_loader), desc='Eval') model.eval() dev_running_loss = 0.0 dev_steps = 0 all_discrete_preds = [] all_labels = [] for batch in dev_loader: batched_sentences, batched_attn, batched_labels = batch if torch.cuda.is_available(): batched_sentences = batched_sentences.cuda() batched_attn = batched_attn.cuda() batched_labels = batched_labels.cuda() outputs = model(batched_sentences, attention_mask=batched_attn) optimizer.zero_grad() dev_loss = criterion(outputs, batched_labels) dev_running_loss += dev_loss.item() dev_steps += 1 discrete_pred = get_discrete_pred( outputs.detach().cpu().numpy(), id2label=id2label) discrete_lab = get_discrete_pred( batched_labels.detach().cpu().numpy(), id2label=id2label) all_discrete_preds.extend(discrete_pred) all_labels.extend(discrete_lab) bar.update(1) bar.close() dev_loss = dev_running_loss / dev_steps avg_pred_number, void_prediction_counter, micro_p, micro_r, micro_f1, macro_p, macro_r, macro_f1 = compute_metrics( all_discrete_preds, all_labels) if epoch == 0: best_macro_f1 = macro_f1 patience = 0 else: if macro_f1 > best_macro_f1: best_macro_f1 = macro_f1 patience = 0 else: patience += 1 if patience >= max_patience: early_stop = True print( 'epoch: {}; loss: {:.2f}; val_loss: {:.2f}; macro_f1: {:.2f}; best_macro_f1: {:.2f}, patience: {}' .format(epoch, loss, dev_loss, macro_f1, best_macro_f1, patience)) if early_stop: print('early stopped') epoch += 1