def forward(self, input_ids,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
labels=None,
input_lens=None):
outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask,
token_type_ids=token_type_ids, position_ids=position_ids, head_mask=head_mask)
last_hidden_state = outputs[0]
sequence_output = self.dropout(last_hidden_state)
logits = self.classifier(sequence_output)
outputs = (logits,) + outputs[2:]
if labels is not None:
assert self.loss_type in ['ce', 'fl', 'lsc']
if self.loss_type == 'ce':
loss_fct = CrossEntropyLoss(ignore_index=0)
elif self.loss_type == 'fl':
loss_fct = FocalLoss(ignore_index=0)
elif self.loss_type == 'lsc':
loss_fct = LabelSmoothingCrossEntropy(ignore_index=0)
if attention_mask is not None:
active_loss = attention_mask.contiguous().view(-1) == 1
active_logits = logits.contiguous().view(-1, self.num_labels)[active_loss]
active_targets = labels.contiguous().view(-1)[active_loss]
loss = loss_fct(active_logits, active_targets)
else:
loss = loss_fct(logits.contiguous().view(-1, self.num_labels), labels.contiguous().view(-1))
outputs = (loss,) + outputs
return outputs
def forward(self, input_ids, attention_mask=None, token_type_ids=None,
position_ids=None, head_mask=None, labels=None):
outputs = self.bert(input_ids = input_ids,attention_mask=attention_mask,token_type_ids=token_type_ids)
sequence_output = outputs[0]
sequence_output = self.dropout(sequence_output)
logits = self.classifier(sequence_output)
outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here
if labels is not None:
assert self.loss_type in ['lsr', 'focal', 'ce']
if self.loss_type == 'lsr':
loss_fct = LabelSmoothingCrossEntropy(ignore_index=0)
elif self.loss_type == 'focal':
loss_fct = FocalLoss(ignore_index=0)
else:
loss_fct = CrossEntropyLoss(ignore_index=0)
# Only keep active parts of the loss
if attention_mask is not None:
active_loss = attention_mask.contiguous().view(-1) == 1
active_logits = logits.view(-1, self.num_labels)[active_loss]
active_labels = labels.contiguous().view(-1)[active_loss]
loss = loss_fct(active_logits, active_labels)
else:
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
outputs = (loss,) + outputs
return outputs # (loss), scores, (hidden_states), (attentions)
def forward(self,
input_ids,
token_type_ids=None,
attention_mask=None,
start_positions=None,
end_positions=None):
outputs = self.bert(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids)
sequence_output = outputs[0]
sequence_output = self.dropout(sequence_output)
start_logits = self.start_fc(sequence_output)
if start_positions is not None and self.training:
if self.soft_label:
batch_size = input_ids.size(0)
seq_len = input_ids.size(1)
label_logits = torch.FloatTensor(batch_size, seq_len,
self.num_labels)
label_logits.zero_()
label_logits = label_logits.to(input_ids.device)
label_logits.scatter_(2, start_positions.unsqueeze(2), 1)
else:
label_logits = start_positions.unsqueeze(2).float()
else:
label_logits = F.softmax(start_logits, -1)
if not self.soft_label:
label_logits = torch.argmax(label_logits,
-1).unsqueeze(2).float()
end_logits = self.end_fc(sequence_output, label_logits)
outputs = (
start_logits,
end_logits,
) + outputs[2:]
if start_positions is not None and end_positions is not None:
assert self.loss_type in ['lsr', 'focal', 'ce']
if self.loss_type == 'lsr':
loss_fct = LabelSmoothingCrossEntropy()
elif self.loss_type == 'focal':
loss_fct = FocalLoss()
else:
loss_fct = CrossEntropyLoss()
start_logits = start_logits.view(-1, self.num_labels)
end_logits = end_logits.view(-1, self.num_labels)
active_loss = attention_mask.view(-1) == 1
active_start_logits = start_logits[active_loss]
active_end_logits = end_logits[active_loss]
active_start_labels = start_positions.view(-1)[active_loss]
active_end_labels = end_positions.view(-1)[active_loss]
start_loss = loss_fct(active_start_logits, active_start_labels)
end_loss = loss_fct(active_end_logits, active_end_labels)
total_loss = (start_loss + end_loss) / 2
outputs = (total_loss, ) + outputs
return outputs
def __init__(self, **kwargs):
"""
Initializes the network
"""
super(LitsSegmentator, self).__init__()
self.args = kwargs
self.optimizer = None
self.model = UNet3D(self.args['n_channels'],
self.args['n_class'],
dropout_val=self.args['dropout_rate'])
class_weights = np.array(
[float(i) for i in self.args['class_weights'].split(',')])
self.criterion = FocalLoss(apply_nonlin=None,
alpha=class_weights,
gamma=2)
self._to_console = False
def forward(self,
input_ids,
attention_mask,
labels,
token_type_ids=None,
input_lens=None):
embs = self.embedding(input_ids)
embs = self.dropout(embs)
embs = embs * attention_mask.float().unsqueeze(2)
sequence_output, _ = self.bilstm(embs)
sequence_output = self.layer_norm(sequence_output)
logits = self.classifier(sequence_output)
outputs = (logits, )
if labels is not None:
if self.use_crf:
loss = self.crf(emissions=logits,
tags=labels,
mask=attention_mask)
outputs = (-1 * loss, ) + outputs
else:
assert self.loss_type in ['ce', 'fl', 'lsc']
if self.loss_type == 'ce':
loss_fct = CrossEntropyLoss(ignore_index=0)
elif self.loss_type == 'fl':
loss_fct = FocalLoss(ignore_index=0)
elif self.loss_type == 'lsc':
loss_fct = LabelSmoothingCrossEntropy(ignore_index=0)
if attention_mask is not None:
active_loss = attention_mask.contiguous().view(-1) == 1
active_logits = logits.contiguous().view(
-1, self.num_labels)[active_loss]
active_targets = labels.contiguous().view(-1)[active_loss]
loss = loss_fct(active_logits, active_targets)
else:
loss = loss_fct(
logits.contiguous().view(-1, self.num_labels),
labels.contiguous().view(-1))
outputs = (loss, ) + outputs
return outputs # (loss), scores
def forward(self, input_ids,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
labels=None,
input_lens=None):
outputs = self.bert(input_ids=input_ids, token_type_ids=token_type_ids,
position_ids=position_ids, head_mask=head_mask, attention_mask=attention_mask)
last_hidden_state = outputs[0] # (batch_size, sequence_length, hidden_size)
if self.use_lstm:
last_hidden_state, _ = self.bilstm(last_hidden_state)
sequence_output = self.dropout(last_hidden_state)
print(sequence_output.shape) # (batch_size, sequence_length, hidden_size)
logits = self.classifier(sequence_output) # (batch_size, seq_length, num_labels)
outputs = (logits,) + outputs
if labels is not None:
if self.use_crf:
loss = self.crf(emissions = logits, tags=labels, mask=attention_mask)
outputs =(-1*loss,)+outputs
else:
assert self.loss_type in ['ce', 'fl', 'lsc']
if self.loss_type == 'ce':
loss_fct = CrossEntropyLoss(ignore_index=0)
elif self.loss_type == 'fl':
loss_fct = FocalLoss(ignore_index=0)
elif self.loss_type == 'lsc':
loss_fct = LabelSmoothingCrossEntropy(ignore_index=0)
if attention_mask is not None:
active_loss = attention_mask.contiguous().view(-1) == 1
active_logits = logits.contiguous().view(-1, self.num_labels)[active_loss]
active_targets = labels.contiguous().view(-1)[active_loss]
loss = loss_fct(active_logits, active_targets)
else:
loss = loss_fct(logits.contiguous().view(-1, self.num_labels), labels.contiguous().view(-1))
outputs = (loss,) + outputs
return outputs
data_transform=VAL_TRANSFORMS,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
pin_memory=True)
MODEL = FurnitureSqueezeNet350(pretrained=True)
N_EPOCHS = 100
OPTIM = Adam(
params=[
{"params": MODEL.features.parameters(), 'lr': 0.0001},
{"params": MODEL.classifier.parameters(), 'lr': 0.001},
],
)
CRITERION = FocalLoss(gamma=2.0)
LR_SCHEDULERS = [
MultiStepLR(OPTIM, milestones=[5, 7, 9, 10, 11, 12, 13], gamma=0.5)
]
REDUCE_LR_ON_PLATEAU = ReduceLROnPlateau(OPTIM, mode='min', factor=0.5, patience=5, threshold=0.05, verbose=True)
EARLY_STOPPING_KWARGS = {
'patience': 30,
# 'score_function': None
}
LOG_INTERVAL = 100
def test_focal_loss(self):
loss = FocalLoss(apply_sigmoid=False)
ret_loss = loss(self.just_input, self.target)
self.assertEqual(17, int(ret_loss.item()))
def test_focal_loss_with_wrong_inut(self):
loss = FocalLoss(apply_sigmoid=False)
ret_loss = loss(self.fully_wrong_input, self.target)
self.assertAlmostEqual(80, ret_loss.item())
def main(config):
model = getattr(encoders,
config['model_name'])(out_features=config['features'],
device=device)
margin = getattr(margins,
config['margin_name'])(in_features=config['features'],
out_features=config['num_classes'],
device=device)
if config['snapshot']['use']:
load_weights(model, config['prefix'], 'model',
config['snapshot']['epoch'])
load_weights(margin, config['prefix'], 'margin',
config['snapshot']['epoch'])
if torch.cuda.is_available() and config['parallel']:
model = nn.DataParallel(model)
margin = nn.DataParallel(margin)
if config['criterion'] == 'FocalLoss':
criterion = FocalLoss(gamma=2)
elif config['criterion'] == 'CrossEntropyLoss':
criterion = nn.CrossEntropyLoss()
if config['optimizer'] == 'SGD':
optimizer = optim.SGD([{
'params': model.parameters()
}, {
'params': margin.parameters()
}],
lr=config['learning_rate'],
momentum=config['momentum'],
weight_decay=config['weight_decay'])
elif config['optimizer'] == 'Adam':
optimizer = optim.Adam([{
'params': model.parameters()
}, {
'params': margin.parameters()
}],
lr=config['learning_rate'],
weight_decay=config['weight_decay'])
lr_scheduler = optim.lr_scheduler.MultiStepLR(
optimizer, milestones=config['milestones'], gamma=0.1)
transforms = Transforms(input_size=config['input_size'], train=True)
data_loader = DataLoader(insightface.Train(
folder=config['train']['folder'],
dataset=config['train']['dataset'],
transforms=transforms),
batch_size=config['batch_size'],
num_workers=config['num_workers'],
shuffle=True)
for epoch in range(
config['snapshot']['epoch'] if config['snapshot']['use'] else 0,
config['num_epochs']):
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
train(data_loader, model, margin, criterion, optimizer, epoch, config)
lr_scheduler.step()
if (epoch + 1) % config['save_freq'] == 0:
save_weights(model, config['prefix'], 'model', epoch + 1,
config['parallel'])
save_weights(margin, config['prefix'], 'margin', epoch + 1,
config['parallel'])
def __init__(self, config):
super(CombinedLoss, self).__init__()
self._focal_loss_indices = config.FOCAL_LOSS_INDICES
self._ce_loss_indices = config.CE_LOSS_INDICES
self._focal_loss = FocalLoss()
self._ce_loss = torch.nn.CrossEntropyLoss()
import torch.nn as nn
from datetime import datetime
from model_zoo.BBSNet.models.BBSNet_model import BBSNet
from model_zoo.BBSNet.models.BBSNet_model_effnet import BBSNet as BBSNet_effnet
from model_zoo.BBSNet import bbs_utils, data
from losses.dice_loss import DiceLoss
from losses.focal_loss import FocalLoss
from losses.jaccard_loss import JaccardLoss
MODELS = {'BBS-Net': BBSNet, 'BBS-Net-Effnet': BBSNet_effnet}
LOSSES = {
'cross-entropy': nn.BCEWithLogitsLoss(),
'dice': DiceLoss(apply_sigmoid=True, smooth=0.0),
'focal': FocalLoss(apply_sigmoid=True, alpha=0.8, gamma=2),
'jaccard': JaccardLoss(apply_sigmoid=True, smooth=0.0)
}
def init_wnb_config(args_):
return dict(wandb_project=args_.wandb_project,
model_name=args_.model,
epochs=args_.epoch,
lr=args_.lr,
input_size=args_.input_size,
batch_size=args_.batch_size,
clip=args_.clip,
decay_rate=args_.decay_rate,
decay_epoch=args_.decay_epoch,
comment=args_.comment or '')
