def __init__(self, hparams): super(MaskRefineModel, self).__init__() self.save_hyperparameters() # Hyperparameters self.hparams = hparams self.lr = hparams.lr # Modules self.net = load_network(hparams) self.criterion = TverskyLoss(hparams.tversky_alpha, hparams.tversky_beta) # Metrics self.train_metrics = MetricCollection([Precision(), Recall(), F1(2)]) self.val_metrics = MetricCollection([Precision(), Recall(), F1(2)])
def __init__(self,MODEL,TRAIN_DATA,TRAIN_CODES,DEV_DATA,DEV_CODES,TEST_DATA,TEST_CODES,HIDDEN_UNIT1,BATCH_SIZE,LR,EPS,EPOCHS,FREEZE_BERT=False): super(CorefClassifier, self).__init__() #self.save_hyperparameters() self.BEST_THRESHOLD = 0 self.train_data = TRAIN_DATA self.train_codes = TRAIN_CODES self.dev_data = DEV_DATA self.dev_codes = DEV_CODES self.test_data = TEST_DATA self.test_codes = TEST_CODES self.model = AutoModel.from_pretrained(MODEL) self.hidden_unit1 = HIDDEN_UNIT1 if self.hidden_unit1: self.hidden_layer1 = nn.Linear(768, self.hidden_unit1) self.hidden_layer2 = nn.Linear(self.hidden_unit1, 1) else: self.hidden_layer1 = nn.Linear(768, 1) self.lossfn = nn.BCELoss() self.batch_size = BATCH_SIZE self.lr = LR self.eps = EPS self.epochs = EPOCHS if FREEZE_BERT: for param in self.model.parameters(): param.requires_grad = False #Metrics self.valid_metrics = MetricCollection([Accuracy(), Precision(num_classes=1, average='macro'), Recall(num_classes=1, average='macro'), F1(num_classes=1, average='macro') ]) self.test_metrics = MetricCollection([Accuracy(), Precision(num_classes=1, average='macro'), Recall(num_classes=1, average='macro'), F1(num_classes=1, average='macro') ])
def test_v1_5_metrics_collection(): target = torch.tensor([0, 2, 0, 2, 0, 1, 0, 2]) preds = torch.tensor([2, 1, 2, 0, 1, 2, 2, 2]) MetricCollection.__init__._warned = False with pytest.deprecated_call(match="It will be removed in v1.5.0."): metrics = MetricCollection([Accuracy()]) assert metrics(preds, target) == {'Accuracy': torch.tensor(0.1250)}
def __init__(self, hparams): super(EdgeCompleteModel, self).__init__() self.save_hyperparameters() # Hyperparameters self.hparams = hparams self.lr = hparams.lr # Modules self.mask_refine_net = MaskRefineNet(hparams.mask_refine_weights) self.mask_refine_net.freeze() self.net = load_network(hparams) self.tv_loss = TverskyLoss(hparams.tversky_alpha, hparams.tversky_beta) # Metrics self.train_metrics = MetricCollection([Precision(), Recall(), F1(2)]) self.val_metrics = MetricCollection([Precision(), Recall(), F1(2)])
def test_v1_5_0_metrics_collection(): target = torch.tensor([0, 2, 0, 2, 0, 1, 0, 2]) preds = torch.tensor([2, 1, 2, 0, 1, 2, 2, 2]) with pytest.deprecated_call( match="The `MetricCollection` was deprecated since v1.3.0 in favor" " of `torchmetrics.collections.MetricCollection`. It will be removed in v1.5.0" ): metrics = MetricCollection([Accuracy()]) assert metrics(preds, target) == {'Accuracy': torch.Tensor([0.1250])[0]}
def make_metric(metric): if metric is None: return None if isinstance(metric, list): m = list(map(instantiate, metric)) elif isinstance(metric, dict): m = {name: instantiate(value) for name, value in metric.items()} else: m = [instantiate(metric)] return MetricCollection(metrics=m)
def test_v1_5_metrics_collection(): target = torch.tensor([0, 2, 0, 2, 0, 1, 0, 2]) preds = torch.tensor([2, 1, 2, 0, 1, 2, 2, 2]) MetricCollection.__init__.warned = False with pytest.deprecated_call( match="`pytorch_lightning.metrics.metric.MetricCollection` was deprecated since v1.3.0 in favor" " of `torchmetrics.collections.MetricCollection`. It will be removed in v1.5.0." ): metrics = MetricCollection([Accuracy()]) assert metrics(preds, target) == {'Accuracy': torch.tensor(0.1250)}
class TestModel(BoringModel): def __init__(self): super().__init__() self.metric = MetricCollection([SumMetric(), DiffMetric()]) self.sum = 0.0 self.diff = 0.0 def training_step(self, batch, batch_idx): x = batch metric_vals = self.metric(x.sum()) self.sum += x.sum() self.diff -= x.sum() self.log_dict({f"{k}_step": v for k, v in metric_vals.items()}) return self.step(x) def training_epoch_end(self, outputs): metric_vals = self.metric.compute() self.log_dict({f"{k}_epoch": v for k, v in metric_vals.items()})
def __init__(self, dm, cfg): """ A model that predicts the name of a variable using its usages. :param dm: datamodule which contains train, val, test datasets, :param cfg: config that contains all needed parameters. """ super(IdTransformerModel, self).__init__() self.dm = dm self.max_sequence_length = cfg.max_sequence_length self.max_num_usages = cfg.max_num_usages self.max_target_length = cfg.max_target_length self.embedding_dim = cfg.embedding_dim self.usage_embedding_dim = cfg.usage_embedding_dim self.target_embedding_dim = cfg.target_embedding_dim self.num_heads = cfg.num_heads self.num_encoder_layers = cfg.num_encoder_layers self.num_decoder_layers = cfg.num_decoder_layers self.num_usage_layers = cfg.num_usage_layers self.dropout = cfg.dropout self.sequence_reducer = cfg.sequence_reducer self.sequence_encoder_type = cfg.sequence_encoder_type self.batch_size = cfg.batch_size self.max_lr = cfg.max_lr self.encoder = IdEncoder( self.max_sequence_length, self.max_num_usages, self.dm.usage_vocab_size, self.dm.usage_pad_idx, self.embedding_dim, self.num_heads, self.usage_embedding_dim, self.num_encoder_layers, self.num_usage_layers, self.dropout, self.sequence_encoder_type, self.sequence_reducer) self.decoder = IdDecoder(self.max_num_usages, self.max_target_length, self.dm.target_vocab_size, self.dm.target_pad_idx, self.usage_embedding_dim, self.num_heads, self.target_embedding_dim, self.num_decoder_layers, self.dropout) self.tusk = "generation" if cfg.target_tokenizer_type is not None else "classification" self.train_metrics = MetricCollection(self.metrics_dict("train")) self.val_metrics = MetricCollection(self.metrics_dict("val")) self.test_metrics = MetricCollection(self.metrics_dict("test")) self.loss = nn.CrossEntropyLoss()
def __init__(self, hparams): super(TextRemovalModel, self).__init__() self.save_hyperparameters() # Hyperparameters self.hparams = hparams self.lr = hparams.lr # Networks self.mask_refine_net = MaskRefineNet(hparams.mask_refine_weights) self.mask_refine_net.freeze() self.edge_complete_net = EdgeCompleteNet(hparams.edge_complete_weights) self.edge_complete_net.freeze() self.net = load_network(hparams) # Losses self.l1_loss = nn.L1Loss() self.gradient_loss = GradientLoss(type=hparams.gradient_loss_type) self.perceptual_loss = PerceptualLoss() # Metrics self.train_psnr = PSNR() self.val_metrics = MetricCollection([PSNR(), MeanAbsoluteError()])
def __init__(self): super().__init__() self.metric = MetricCollection([SumMetric(), DiffMetric()]) self.sum = 0.0 self.diff = 0.0
class IdTransformerModel(pl.LightningModule): def __init__(self, dm, cfg): """ A model that predicts the name of a variable using its usages. :param dm: datamodule which contains train, val, test datasets, :param cfg: config that contains all needed parameters. """ super(IdTransformerModel, self).__init__() self.dm = dm self.max_sequence_length = cfg.max_sequence_length self.max_num_usages = cfg.max_num_usages self.max_target_length = cfg.max_target_length self.embedding_dim = cfg.embedding_dim self.usage_embedding_dim = cfg.usage_embedding_dim self.target_embedding_dim = cfg.target_embedding_dim self.num_heads = cfg.num_heads self.num_encoder_layers = cfg.num_encoder_layers self.num_decoder_layers = cfg.num_decoder_layers self.num_usage_layers = cfg.num_usage_layers self.dropout = cfg.dropout self.sequence_reducer = cfg.sequence_reducer self.sequence_encoder_type = cfg.sequence_encoder_type self.batch_size = cfg.batch_size self.max_lr = cfg.max_lr self.encoder = IdEncoder( self.max_sequence_length, self.max_num_usages, self.dm.usage_vocab_size, self.dm.usage_pad_idx, self.embedding_dim, self.num_heads, self.usage_embedding_dim, self.num_encoder_layers, self.num_usage_layers, self.dropout, self.sequence_encoder_type, self.sequence_reducer) self.decoder = IdDecoder(self.max_num_usages, self.max_target_length, self.dm.target_vocab_size, self.dm.target_pad_idx, self.usage_embedding_dim, self.num_heads, self.target_embedding_dim, self.num_decoder_layers, self.dropout) self.tusk = "generation" if cfg.target_tokenizer_type is not None else "classification" self.train_metrics = MetricCollection(self.metrics_dict("train")) self.val_metrics = MetricCollection(self.metrics_dict("val")) self.test_metrics = MetricCollection(self.metrics_dict("test")) self.loss = nn.CrossEntropyLoss() def metrics_dict(self, prefix="train"): if self.tusk == "classification": return { f"{prefix}_top1": Top(n=1), f"{prefix}_top5": Top(n=5), f"{prefix}_MRR": MRR() } elif self.tusk == "generation": ignore_idxs = (self.dm.target_eos_idx, self.dm.target_pad_idx) return { f"{prefix}_accuracy": Accuracy(), f"{prefix}_precision": Precision(ignore_idxs), f"{prefix}_recall": Recall(ignore_idxs), f"{prefix}_F1": F1(ignore_idxs) } else: return ValueError(f"{self.tusk} tusk is not supported") def forward(self, usages, num_usages=None, beam_search=True, limit_num_usages=10, **kwargs): inp = usages[:limit_num_usages] with torch.no_grad(): memory = self.encoder(inp.to(self.device), num_usages) if beam_search: out = self.decoder.beam_search(memory, self.max_target_length, self.dm.target_init_idx, self.dm.target_eos_idx, self.dm.target_pad_idx, **kwargs) else: out = self.decoder.greedy_search(memory, self.max_target_length, self.dm.target_init_idx, self.dm.target_eos_idx) return out def training_step(self, batch, batch_idx): loss, predictions, targets = self.step(batch, batch_idx) self.log('train_loss', loss, prog_bar=True, on_step=True, on_epoch=True) self.train_metrics.update(predictions, targets) self.log_dict(self.train_metrics, on_step=False, on_epoch=True, prog_bar=False) return loss def validation_step(self, batch, batch_idx): loss, predictions, targets = self.step(batch, batch_idx) self.log('val_loss', loss, prog_bar=True, on_step=True, on_epoch=True) self.val_metrics.update(predictions, targets) self.log_dict(self.val_metrics, on_step=False, on_epoch=True, prog_bar=False) return loss def test_step(self, batch, batch_idx): loss, predictions, targets = self.step(batch, batch_idx) self.log('test_loss', loss, prog_bar=True, on_step=True, on_epoch=True) self.test_metrics.update(predictions, targets) self.log_dict(self.test_metrics, on_step=False, on_epoch=True, prog_bar=False) def step(self, batch, batch_idx): src, tgt = batch x, num_usages, _ = src # BxUxL, B tgt, tgt_length = tgt # TxB, B memory = self.encoder(x, num_usages) out = self.decoder(memory, tgt, num_usages, tgt_length) tgt_to_loss = torch.constant_pad_nd(tgt, (0, 0, 0, 1), self.dm.target_pad_idx)[1:, ...] # TxB # [[<s>], [<token>], [</s>], [<pad>]]] -> [[<token>], [</s>], [<pad>], [<pad>]] out_to_loss = out.transpose(1, 2) # TxBxV -> TxVxB return self.loss(out_to_loss, tgt_to_loss), out, tgt # 1, TxBxV, TxB def configure_optimizers(self): def schedule(step, warmup_steps=len(self.dm.trainloader)): x = (step + 1) / warmup_steps return min(1, x**(-0.5)) optimizer = torch.optim.Adam(self.parameters(), lr=self.max_lr) scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, schedule) return { "optimizer": optimizer, 'scheduler': scheduler, 'interval': 'step' }