def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, labels=None, input_lens=None): outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, head_mask=head_mask) last_hidden_state = outputs[0] sequence_output = self.dropout(last_hidden_state) logits = self.classifier(sequence_output) outputs = (logits,) + outputs[2:] if labels is not None: assert self.loss_type in ['ce', 'fl', 'lsc'] if self.loss_type == 'ce': loss_fct = CrossEntropyLoss(ignore_index=0) elif self.loss_type == 'fl': loss_fct = FocalLoss(ignore_index=0) elif self.loss_type == 'lsc': loss_fct = LabelSmoothingCrossEntropy(ignore_index=0) if attention_mask is not None: active_loss = attention_mask.contiguous().view(-1) == 1 active_logits = logits.contiguous().view(-1, self.num_labels)[active_loss] active_targets = labels.contiguous().view(-1)[active_loss] loss = loss_fct(active_logits, active_targets) else: loss = loss_fct(logits.contiguous().view(-1, self.num_labels), labels.contiguous().view(-1)) outputs = (loss,) + outputs return outputs
def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, labels=None): outputs = self.bert(input_ids = input_ids,attention_mask=attention_mask,token_type_ids=token_type_ids) sequence_output = outputs[0] sequence_output = self.dropout(sequence_output) logits = self.classifier(sequence_output) outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here if labels is not None: assert self.loss_type in ['lsr', 'focal', 'ce'] if self.loss_type == 'lsr': loss_fct = LabelSmoothingCrossEntropy(ignore_index=0) elif self.loss_type == 'focal': loss_fct = FocalLoss(ignore_index=0) else: loss_fct = CrossEntropyLoss(ignore_index=0) # Only keep active parts of the loss if attention_mask is not None: active_loss = attention_mask.contiguous().view(-1) == 1 active_logits = logits.view(-1, self.num_labels)[active_loss] active_labels = labels.contiguous().view(-1)[active_loss] loss = loss_fct(active_logits, active_labels) else: loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) outputs = (loss,) + outputs return outputs # (loss), scores, (hidden_states), (attentions)
def forward(self, input_ids, token_type_ids=None, attention_mask=None, start_positions=None, end_positions=None): outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids) sequence_output = outputs[0] sequence_output = self.dropout(sequence_output) start_logits = self.start_fc(sequence_output) if start_positions is not None and self.training: if self.soft_label: batch_size = input_ids.size(0) seq_len = input_ids.size(1) label_logits = torch.FloatTensor(batch_size, seq_len, self.num_labels) label_logits.zero_() label_logits = label_logits.to(input_ids.device) label_logits.scatter_(2, start_positions.unsqueeze(2), 1) else: label_logits = start_positions.unsqueeze(2).float() else: label_logits = F.softmax(start_logits, -1) if not self.soft_label: label_logits = torch.argmax(label_logits, -1).unsqueeze(2).float() end_logits = self.end_fc(sequence_output, label_logits) outputs = ( start_logits, end_logits, ) + outputs[2:] if start_positions is not None and end_positions is not None: assert self.loss_type in ['lsr', 'focal', 'ce'] if self.loss_type == 'lsr': loss_fct = LabelSmoothingCrossEntropy() elif self.loss_type == 'focal': loss_fct = FocalLoss() else: loss_fct = CrossEntropyLoss() start_logits = start_logits.view(-1, self.num_labels) end_logits = end_logits.view(-1, self.num_labels) active_loss = attention_mask.view(-1) == 1 active_start_logits = start_logits[active_loss] active_end_logits = end_logits[active_loss] active_start_labels = start_positions.view(-1)[active_loss] active_end_labels = end_positions.view(-1)[active_loss] start_loss = loss_fct(active_start_logits, active_start_labels) end_loss = loss_fct(active_end_logits, active_end_labels) total_loss = (start_loss + end_loss) / 2 outputs = (total_loss, ) + outputs return outputs
def __init__(self, **kwargs): """ Initializes the network """ super(LitsSegmentator, self).__init__() self.args = kwargs self.optimizer = None self.model = UNet3D(self.args['n_channels'], self.args['n_class'], dropout_val=self.args['dropout_rate']) class_weights = np.array( [float(i) for i in self.args['class_weights'].split(',')]) self.criterion = FocalLoss(apply_nonlin=None, alpha=class_weights, gamma=2) self._to_console = False
def forward(self, input_ids, attention_mask, labels, token_type_ids=None, input_lens=None): embs = self.embedding(input_ids) embs = self.dropout(embs) embs = embs * attention_mask.float().unsqueeze(2) sequence_output, _ = self.bilstm(embs) sequence_output = self.layer_norm(sequence_output) logits = self.classifier(sequence_output) outputs = (logits, ) if labels is not None: if self.use_crf: loss = self.crf(emissions=logits, tags=labels, mask=attention_mask) outputs = (-1 * loss, ) + outputs else: assert self.loss_type in ['ce', 'fl', 'lsc'] if self.loss_type == 'ce': loss_fct = CrossEntropyLoss(ignore_index=0) elif self.loss_type == 'fl': loss_fct = FocalLoss(ignore_index=0) elif self.loss_type == 'lsc': loss_fct = LabelSmoothingCrossEntropy(ignore_index=0) if attention_mask is not None: active_loss = attention_mask.contiguous().view(-1) == 1 active_logits = logits.contiguous().view( -1, self.num_labels)[active_loss] active_targets = labels.contiguous().view(-1)[active_loss] loss = loss_fct(active_logits, active_targets) else: loss = loss_fct( logits.contiguous().view(-1, self.num_labels), labels.contiguous().view(-1)) outputs = (loss, ) + outputs return outputs # (loss), scores
def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, labels=None, input_lens=None): outputs = self.bert(input_ids=input_ids, token_type_ids=token_type_ids, position_ids=position_ids, head_mask=head_mask, attention_mask=attention_mask) last_hidden_state = outputs[0] # (batch_size, sequence_length, hidden_size) if self.use_lstm: last_hidden_state, _ = self.bilstm(last_hidden_state) sequence_output = self.dropout(last_hidden_state) print(sequence_output.shape) # (batch_size, sequence_length, hidden_size) logits = self.classifier(sequence_output) # (batch_size, seq_length, num_labels) outputs = (logits,) + outputs if labels is not None: if self.use_crf: loss = self.crf(emissions = logits, tags=labels, mask=attention_mask) outputs =(-1*loss,)+outputs else: assert self.loss_type in ['ce', 'fl', 'lsc'] if self.loss_type == 'ce': loss_fct = CrossEntropyLoss(ignore_index=0) elif self.loss_type == 'fl': loss_fct = FocalLoss(ignore_index=0) elif self.loss_type == 'lsc': loss_fct = LabelSmoothingCrossEntropy(ignore_index=0) if attention_mask is not None: active_loss = attention_mask.contiguous().view(-1) == 1 active_logits = logits.contiguous().view(-1, self.num_labels)[active_loss] active_targets = labels.contiguous().view(-1)[active_loss] loss = loss_fct(active_logits, active_targets) else: loss = loss_fct(logits.contiguous().view(-1, self.num_labels), labels.contiguous().view(-1)) outputs = (loss,) + outputs return outputs
data_transform=VAL_TRANSFORMS, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=True) MODEL = FurnitureSqueezeNet350(pretrained=True) N_EPOCHS = 100 OPTIM = Adam( params=[ {"params": MODEL.features.parameters(), 'lr': 0.0001}, {"params": MODEL.classifier.parameters(), 'lr': 0.001}, ], ) CRITERION = FocalLoss(gamma=2.0) LR_SCHEDULERS = [ MultiStepLR(OPTIM, milestones=[5, 7, 9, 10, 11, 12, 13], gamma=0.5) ] REDUCE_LR_ON_PLATEAU = ReduceLROnPlateau(OPTIM, mode='min', factor=0.5, patience=5, threshold=0.05, verbose=True) EARLY_STOPPING_KWARGS = { 'patience': 30, # 'score_function': None } LOG_INTERVAL = 100
def test_focal_loss(self): loss = FocalLoss(apply_sigmoid=False) ret_loss = loss(self.just_input, self.target) self.assertEqual(17, int(ret_loss.item()))
def test_focal_loss_with_wrong_inut(self): loss = FocalLoss(apply_sigmoid=False) ret_loss = loss(self.fully_wrong_input, self.target) self.assertAlmostEqual(80, ret_loss.item())
def main(config): model = getattr(encoders, config['model_name'])(out_features=config['features'], device=device) margin = getattr(margins, config['margin_name'])(in_features=config['features'], out_features=config['num_classes'], device=device) if config['snapshot']['use']: load_weights(model, config['prefix'], 'model', config['snapshot']['epoch']) load_weights(margin, config['prefix'], 'margin', config['snapshot']['epoch']) if torch.cuda.is_available() and config['parallel']: model = nn.DataParallel(model) margin = nn.DataParallel(margin) if config['criterion'] == 'FocalLoss': criterion = FocalLoss(gamma=2) elif config['criterion'] == 'CrossEntropyLoss': criterion = nn.CrossEntropyLoss() if config['optimizer'] == 'SGD': optimizer = optim.SGD([{ 'params': model.parameters() }, { 'params': margin.parameters() }], lr=config['learning_rate'], momentum=config['momentum'], weight_decay=config['weight_decay']) elif config['optimizer'] == 'Adam': optimizer = optim.Adam([{ 'params': model.parameters() }, { 'params': margin.parameters() }], lr=config['learning_rate'], weight_decay=config['weight_decay']) lr_scheduler = optim.lr_scheduler.MultiStepLR( optimizer, milestones=config['milestones'], gamma=0.1) transforms = Transforms(input_size=config['input_size'], train=True) data_loader = DataLoader(insightface.Train( folder=config['train']['folder'], dataset=config['train']['dataset'], transforms=transforms), batch_size=config['batch_size'], num_workers=config['num_workers'], shuffle=True) for epoch in range( config['snapshot']['epoch'] if config['snapshot']['use'] else 0, config['num_epochs']): print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) train(data_loader, model, margin, criterion, optimizer, epoch, config) lr_scheduler.step() if (epoch + 1) % config['save_freq'] == 0: save_weights(model, config['prefix'], 'model', epoch + 1, config['parallel']) save_weights(margin, config['prefix'], 'margin', epoch + 1, config['parallel'])
def __init__(self, config): super(CombinedLoss, self).__init__() self._focal_loss_indices = config.FOCAL_LOSS_INDICES self._ce_loss_indices = config.CE_LOSS_INDICES self._focal_loss = FocalLoss() self._ce_loss = torch.nn.CrossEntropyLoss()
import torch.nn as nn from datetime import datetime from model_zoo.BBSNet.models.BBSNet_model import BBSNet from model_zoo.BBSNet.models.BBSNet_model_effnet import BBSNet as BBSNet_effnet from model_zoo.BBSNet import bbs_utils, data from losses.dice_loss import DiceLoss from losses.focal_loss import FocalLoss from losses.jaccard_loss import JaccardLoss MODELS = {'BBS-Net': BBSNet, 'BBS-Net-Effnet': BBSNet_effnet} LOSSES = { 'cross-entropy': nn.BCEWithLogitsLoss(), 'dice': DiceLoss(apply_sigmoid=True, smooth=0.0), 'focal': FocalLoss(apply_sigmoid=True, alpha=0.8, gamma=2), 'jaccard': JaccardLoss(apply_sigmoid=True, smooth=0.0) } def init_wnb_config(args_): return dict(wandb_project=args_.wandb_project, model_name=args_.model, epochs=args_.epoch, lr=args_.lr, input_size=args_.input_size, batch_size=args_.batch_size, clip=args_.clip, decay_rate=args_.decay_rate, decay_epoch=args_.decay_epoch, comment=args_.comment or '')