def __init__(self, gpt2, num_labels):
super().__init__()
self.gpt2 = gpt2
vocab_size = self.gpt2.vocab_size
self.linear = nn.Linear(vocab_size, num_labels)
self.logit = nn.Sigmoid()
self.criterion = nn.BCELoss()
self.train_f1 = F1(mdmc_average='global')
self.valid_f1 = F1(mdmc_average='global')
def __init__(self, num_labels):
super().__init__()
self.hidden_size = 768
self.num_labels = num_labels
self.linear = nn.Linear(self.hidden_size, self.num_labels)
self.train_loss = nn.CrossEntropyLoss()
self.train_f1 = F1()
self.validation_f1 = F1()
self.transformer = AutoModel.from_pretrained(BERT_MODEL_NAME)
def __init__(self,
bert_model,
n_classes: int,
n_training_steps=None,
n_warmup_steps=None):
super().__init__()
self.roberta = RobertaForABSA.from_pretrained(bert_model)
self.classifier = nn.Linear(self.roberta.config.hidden_size, n_classes)
self.n_training_steps = n_training_steps
self.n_warmup_steps = n_warmup_steps
self.criterion = nn.BCELoss()
self.train_f1 = F1(mdmc_average='global')
self.val_f1 = F1(mdmc_average='global')
self.test_f1 = F1(mdmc_average='global')
def __init__(
self,
arch: str,
optcfg: DictConfig,
arch_ckpt: Optional[str] = None,
schcfg: Optional[DictConfig] = None,
**kwargs,
):
super().__init__()
self.schcfg = schcfg
self.optcfg = optcfg
self.save_hyperparameters()
if arch_ckpt:
arch = arch_ckpt
self.transformer = AutoModelForSequenceClassification.from_pretrained(arch, num_labels=7)
# loss function
self.criterion = nn.CrossEntropyLoss()
# metrics
mc = MetricCollection({
"accuracy": Accuracy(threshold=0.0),
"recall": Recall(threshold=0.0, num_classes=7, average='macro'),
"precision": Precision(threshold=0.0, num_classes=7, average='macro'),
"f1": F1(threshold=0.0, num_classes=7, average='macro'),
"macro_auc": AUROC(num_classes=7, average='macro'),
# "weighted_auc": AUROC(num_classes=7, average='weighted')
})
self.metrics: ModuleDict[str, MetricCollection] = ModuleDict({
f"{phase}_metric": mc.clone()
for phase in ["train", "valid", "test"]
})
def main(model_path, data_path, dataset):
# split the training dataset and initialize the data loaders
bs = 5
if dataset == 'TEE':
test_dataset = DatasetTEE(data_path / 'train_gray',
data_path / 'train_gt')
elif dataset == 'CAMUS':
test_dataset = DatasetCAMUS(data_path,
isotropic=False,
include_es=False,
include_2ch=False,
include_4ch=True,
start=401,
stop=450)
test_dl = DataLoader(test_dataset, batch_size=bs, shuffle=False)
# build the Unet2D with one channel as input and 4 channels as output
dice_function = F1(num_classes=4,
average='macro',
ignore_index=0,
mdmc_average='samplewise')
unet = Unet2D(1, 4)
# Load best model params
unet.load_state_dict(torch.load(model_path))
acc, dice = test(unet,
test_dl,
acc_metric,
dice_function,
visual_debug=True)
print('acc: ', acc, 'dice: ', dice)
def main(args):
# Setup experiment
if not torch.cuda.is_available():
raise NotImplementedError("Training on CPU is not supported.")
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
args.experiment = args.experiment or f"{args.model.replace('_', '-')}"
args.experiment = "-".join([args.experiment])
args.experiment_dir = os.path.join(
args.output_dir, args.dataset,
args.experiment + ("-cumulative" if args.all else ""))
os.makedirs(args.experiment_dir, exist_ok=True)
tb_writer = TensorBoardLogger(save_dir=args.experiment_dir)
seed_everything(42, workers=True)
train_loaders, valid_loaders = data.build_dataset(
args.dataset,
args.data_path,
blurry=args.blurry,
batch_size=args.batch_size)
model = GCL(args, train_loaders)
trainer = Trainer(gpus=-1,
distributed_backend='ddp',
max_epochs=len(valid_loaders),
reload_dataloaders_every_epoch=True,
plugins=DDPPlugin(find_unused_parameters=False),
logger=tb_writer)
# for task_id in range(num_tasks):
# print(f"task: {task_id}")
# train_loader = DataLoader(train_loaders[task_id], batch_size=10)
metrics = MetricCollection(
[Accuracy(), F1(args.num_classes, average='macro')])
model.train_metrics = metrics.clone(prefix=f'train{model.curr_index}_')
trainer.fit(model)
trainer.train_loop.reset_train_val_dataloaders(model)
if model.curr_index:
temp = trainer.train_dataloader
labels = []
for batch in temp:
labels.append(batch[1])
print("Check if train loader if reloaded", labels)
#! TEST
test_loaders = [
DataLoader(ds, batch_size=len(ds))
for ds in valid_loaders[:model.curr_index + 1]
]
model.test_metrics = nn.ModuleList(
[metrics.clone(prefix=f'valid{i}_') for i in range(len(test_loaders))])
trainer.test(model, test_dataloaders=test_loaders)
def configure_metrics(self, _) -> None:
self.prec = Precision(num_classes=self.num_labels)
self.recall = Recall(num_classes=self.num_labels)
self.f1 = F1(num_classes=self.num_labels)
self.acc = Accuracy()
self.metrics = {
"precision": self.prec,
"recall": self.recall,
"accuracy": self.acc,
"f1": self.f1
}
def _validate(self, criterion):
"""Validation step"""
self.model.eval()
f1 = F1(num_classes=len(self.data.labels))
bar = tqdm(self.data.val_loader, file=sys.stdout, leave=False)
with torch.no_grad():
for batch in bar:
data, target = batch['text'], batch['target']
data = data['ids'].to('cuda')
output = self.model(data).logits.detach().cpu()
f1.update(output.sigmoid(), target)
return f1.compute()
def get_metrics(metric_threshold, monitor_metrics, num_classes):
macro_prec = Precision(num_classes, metric_threshold, average='macro')
macro_recall = Recall(num_classes, metric_threshold, average='macro')
another_macro_f1 = 2 * (macro_prec * macro_recall) / (macro_prec +
macro_recall + 1e-10)
metrics = {
'Micro-Precision':
Precision(num_classes, metric_threshold, average='micro'),
'Micro-Recall':
Recall(num_classes, metric_threshold, average='micro'),
'Micro-F1':
F1(num_classes, metric_threshold, average='micro'),
'Macro-F1':
F1(num_classes, metric_threshold, average='macro'),
# The f1 value of macro_precision and macro_recall. This variant of
# macro_f1 is less preferred but is used in some works. Please
# refer to Opitz et al. 2019 [https://arxiv.org/pdf/1911.03347.pdf]
'Another-Macro-F1':
another_macro_f1,
}
for metric in monitor_metrics:
if isinstance(metric, Metric): # customized metric
metrics[type(metric).__name__] = metric
elif re.match('P@\d+', metric):
metrics[metric] = Precision(num_classes,
average='samples',
top_k=int(metric[2:]))
elif re.match('R@\d+', metric):
metrics[metric] = Recall(num_classes,
average='samples',
top_k=int(metric[2:]))
elif metric not in [
'Micro-Precision', 'Micro-Recall', 'Micro-F1', 'Macro-F1',
'Another-Macro-F1'
]:
raise ValueError(f'Invalid metric: {metric}')
return MetricCollection(metrics)
def __init__(
self,
n_channel: int = 1,
n_class: int = 2,
learning_rate: float = 1e-4,
class_weight: List[float] = None,
backbone: Union[str, nn.Module] = "simple-cnn",
backbone_output_size: int = 0,
n_hidden: int = 512,
dropout: float = 0.2,
lr_scheduler: bool = False,
lr_scheduler_warmup_steps: int = 100,
lr_scheduler_total_steps: int = 0,
**kwargs,
):
super().__init__()
self.save_hyperparameters()
if isinstance(backbone, str):
self.backbone, backbone_output_size = get_backbone(
backbone,
channels=n_channel,
dropout=dropout,
**kwargs,
)
self.classifier = Classifier(backbone_output_size, n_class, n_hidden,
dropout)
if class_weight is not None:
class_weight = torch.tensor(class_weight, dtype=torch.float)
self.loss_fn = nn.CrossEntropyLoss(weight=class_weight)
self.train_accuracy = Accuracy()
self.val_accuracy = Accuracy()
self.test_metrics = MetricCollection([
Accuracy(),
F1(num_classes=self.hparams.n_class, average="macro"),
Recall(num_classes=self.hparams.n_class,
average="macro"), # balanced acc.
StatScores(
num_classes=self.hparams.n_class
if self.hparams.n_class > 2 else 1,
reduce="micro",
multiclass=self.hparams.n_class > 2,
),
])
def __init__(self, hyp_params, target_names, early_stopping):
super().__init__()
self.model = MULTModel(hyp_params)
self.save_hyperparameters(hyp_params)
self.learning_rate = hyp_params.lr
self.weight_decay = hyp_params.weight_decay
self.target_names = target_names
self.mae_1 = 1 - MeanAbsoluteError()
self.acc2 = Accuracy()
self.acc7 = Accuracy(multiclass=True)
self.f1 = F1()
self.loss = loss_dict[hyp_params.loss_fnc]
self.opt = opt_dict[hyp_params.optim]
self.early_stopping = early_stopping
def test_f1_with_sklearn_and_pl(self): pred = torch.rand(10) target = torch.rand(10).ge(0.5).float() thres = Thresholding(0.5) pred = thres(pred) f1 = FScore() f1_pl = F1(num_classes=10, threshold=0.5, multilabel=True) mlu_score = f1(pred, target).item() sk_score = f1_score(y_pred=pred.cpu().numpy(), y_true=target.cpu().numpy()) pl_score = f1_pl(pred, target).item() # print(pred, target) self.assertAlmostEqual(mlu_score, sk_score) self.assertAlmostEqual(mlu_score, pl_score)
def __init__(self,
hparams,
train_subjects,
validate_subjects,
class_weights=None):
super(TrainRTBENE, self).__init__()
assert class_weights is not None, "Class Weights can't be None"
self.model = MODELS[hparams.model_base]()
self._criterion = torch.nn.BCEWithLogitsLoss(
pos_weight=torch.Tensor([class_weights[1]]))
self._train_subjects = train_subjects
self._validate_subjects = validate_subjects
self._metrics = MetricCollection(
[Accuracy(),
F1(), Precision(),
Recall(), Specificity()])
self.save_hyperparameters(
hparams,
ignore=["train_subjects", "validate_subjects", "class_weights"])
datamodule = ImageClassificationData.from_files(
train_files=train_files,
train_targets=train_targets,
test_files=test_files,
test_targets=test_targets,
val_split=0.1, # Use 10 % of the train dataset to generate validation one.
image_size=(128, 128),
)
# 3. Build the model
model = ImageClassifier(
backbone="resnet18",
num_classes=len(genres),
multi_label=True,
metrics=F1(num_classes=len(genres)),
)
# 4. Create the trainer. Train on 2 gpus for 10 epochs.
trainer = flash.Trainer(max_epochs=10)
# 5. Train the model
trainer.finetune(model, datamodule=datamodule, strategy="freeze")
# 6. Predict what's on a few images!
# Serialize predictions as labels, low threshold to see more predictions.
model.serializer = Labels(genres, multi_label=True, threshold=0.25)
predictions = model.predict([
"data/movie_posters/predict/tt0085318.jpg",
"data/movie_posters/predict/tt0089461.jpg",
"""
new_model = create_model(model.__class__, device)
source_params = dict(model.named_parameters())
source_buffer = dict(model.named_buffers())
for name, param in new_model.named_parameters():
param.data.copy_(source_params[name].data)
for name, buffer_ in new_model.named_buffers():
buffer_.data.copy_(source_buffer[name].data)
return new_model
metrics = MetricCollection([
Accuracy(),
Precision(),
Recall(),
F1(),
])
def train(model: nn.Module,
train_dataloader: DataLoader,
lr: float = 1e-3,
device: str = 'cuda:0',
fast_dev_run=False,
verbose=True) -> Dict[str, torch.Tensor]:
optimizer = torch.optim.Adam(lr=lr, params=model.parameters())
loss_fn = nn.CrossEntropyLoss()
num_batch = len(train_dataloader)
global metrics
datamodule = TextClassificationData.from_csv(
"comment_text",
["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"],
train_file="data/jigsaw_toxic_comments/train.csv",
test_file="data/jigsaw_toxic_comments/test.csv",
predict_file="data/jigsaw_toxic_comments/predict.csv",
batch_size=16,
val_split=0.1,
backbone="unitary/toxic-bert",
)
# 3. Build the model
model = TextClassifier(
num_classes=datamodule.num_classes,
multi_label=True,
metrics=F1(num_classes=datamodule.num_classes),
backbone="unitary/toxic-bert",
)
# 4. Create the trainer
trainer = flash.Trainer(fast_dev_run=True)
# 5. Fine-tune the model
trainer.finetune(model, datamodule=datamodule, strategy="freeze")
# 6. Generate predictions for a few comments!
predictions = model.predict([
"No, he is an arrogant, self serving, immature idiot. Get it right.",
"U SUCK HANNAH MONTANA",
"Would you care to vote? Thx.",
])
def main(
visual_debug=False,
params_path=None,
bs=4,
epochs_val=10,
learn_rate=0.01,
ckpt=None,
dataset='CAMUS',
outputs=4,
pre_process=None,
transform=None,
isotropic=False,
include_es=False,
is_local=False,
include_2ch=False,
include_4ch=False
):
# enable if you want to see some plotting
# load the training data
train_dataset, valid_dataset, _ = get_train_val_set(dataset, is_local, pre_process, transform, isotropic, include_es, include_2ch, include_4ch)
train_dl = DataLoader(train_dataset, batch_size=bs, shuffle=True)
valid_dl = DataLoader(valid_dataset, batch_size=bs, shuffle=False)
print(f'Size of train dataset: {len(train_dataset)}\nSize of validation dataset: {len(valid_dataset)}')
# if visual_debug:
# fig, ax = plt.subplots(1, 2)
# ax[0].imshow(train_dataset.open_as_array(150))
# mask = train_dataset.open_mask(150)
# ax[1].imshow(mask)
# plt.show()
xb, yb = next(iter(train_dl))
#print('c', xb.shape, yb.shape)
# build the Unet2D with one channel as input and 4 channels as output
unet = ImprovedUnet2D(1, outputs)
if ckpt:
unet.load_state_dict(torch.load(ckpt))
# loss function and optimizer
# loss_fn = nn.CrossEntropyLoss()
loss_fn = DiceLoss()
opt = torch.optim.Adam(unet.parameters(), lr=learn_rate)
#opt = torch.optim.SGD(unet.parameters(), lr=learn_rate, momentum=0.9)
dice_function = F1(num_classes=4, average='macro', ignore_index=0, mdmc_average='samplewise')
train_loss, valid_loss = train(unet, train_dl, valid_dl, loss_fn, opt, acc_metric, dice_function, epochs=epochs_val,
params_path=params_path)
# plot training and validation losses
if visual_debug:
fig = plt.figure(figsize=(10, 8))
plt.plot(train_loss, label='Train loss')
plt.plot(valid_loss, label='Valid loss')
plt.legend()
# plt.show()
fig.savefig(params_path/'loss.png')
# predict on the next train batch (is this fair?)
xb, yb = next(iter(train_dl))
with torch.no_grad():
predb = unet(xb.cuda())
# show the predicted segmentations
if visual_debug:
fig, ax = plt.subplots(bs, 3, figsize=(15, bs * 5))
for i in range(bs):
ax[i, 0].imshow(batch_to_img(xb, i))
ax[i, 1].imshow(yb[i])
ax[i, 2].imshow(predb_to_mask(predb, i))
#plt.show()
fig.savefig(params_path/'predictions_final.png')
