def _train(self, loader): self.model.train() bar, metric = progress_bar(loader), AttachmentMetric() for words, feats, arcs, sibs, rels in bar: self.optimizer.zero_grad() mask = words.ne(self.WORD.pad_index) # ignore the first token of each sentence mask[:, 0] = 0 s_arc, s_sib, s_rel = self.model(words, feats) loss, s_arc = self.model.loss(s_arc, s_sib, s_rel, arcs, sibs, rels, mask, self.args.mbr, self.args.partial) loss.backward() nn.utils.clip_grad_norm_(self.model.parameters(), self.args.clip) self.optimizer.step() self.scheduler.step() arc_preds, rel_preds = self.model.decode(s_arc, s_sib, s_rel, mask) if self.args.partial: mask &= arcs.ge(0) # ignore all punctuation if not specified if not self.args.punct: mask &= words.unsqueeze(-1).ne(self.puncts).all(-1) metric(arc_preds, rel_preds, arcs, rels, mask) bar.set_postfix_str(f"lr: {self.scheduler.get_last_lr()[0]:.4e} - loss: {loss:.4f} - {metric}")
def _evaluate(self, loader): self.model.eval() total_loss, metric = 0, AttachmentMetric() for words, feats, arcs, rels in loader: mask = words.ne(self.WORD.pad_index) # ignore the first token of each sentence mask[:, 0] = 0 s_arc, s_rel = self.model(words, feats) loss, s_arc = self.model.loss(s_arc, s_rel, arcs, rels, mask, self.args.mbr, self.args.partial) arc_preds, rel_preds = self.model.decode(s_arc, s_rel, mask, self.args.tree, self.args.proj) if self.args.partial: mask &= arcs.ge(0) # ignore all punctuation if not specified if not self.args.punct: mask &= words.unsqueeze(-1).ne(self.puncts).all(-1) total_loss += loss.item() metric(arc_preds, rel_preds, arcs, rels, mask) total_loss /= len(loader) return total_loss, metric
def _train(self, loader): self.model.train() bar, metric = progress_bar(loader), AttachmentMetric() for i, (words, texts, *feats, arcs, rels) in enumerate(bar, 1): word_mask = words.ne(self.args.pad_index) mask = word_mask if len(words.shape) < 3 else word_mask.any(-1) # ignore the first token of each sentence mask[:, 0] = 0 s_arc, s_sib, s_rel = self.model(words, feats) loss, s_arc = self.model.loss(s_arc, s_sib, s_rel, arcs, rels, mask) loss = loss / self.args.update_steps loss.backward() nn.utils.clip_grad_norm_(self.model.parameters(), self.args.clip) if i % self.args.update_steps == 0: self.optimizer.step() self.scheduler.step() self.optimizer.zero_grad() arc_preds, rel_preds = self.model.decode(s_arc, s_rel, mask) if self.args.partial: mask &= arcs.ge(0) # ignore all punctuation if not specified if not self.args.punct: mask.masked_scatter_(mask, ~mask.new_tensor([ispunct(w) for s in texts for w in s])) metric(arc_preds, rel_preds, arcs, rels, mask) bar.set_postfix_str(f"lr: {self.scheduler.get_last_lr()[0]:.4e} - loss: {loss:.4f} - {metric}") logger.info(f"{bar.postfix}")
def _train(self, loader): self.model.train() bar, metric = progress_bar(loader), AttachmentMetric() # words, feats, etc. come from loader! loader is train.loader, where train is Dataset for words, feats, arcs, rels in bar: self.optimizer.zero_grad() if self.elmo: feat_embs = self.elmo.embed_batch(feats) else: feat_embs = self.efml.sents2elmo(feats, output_layer=-2) #TODO: dodaj mapping, ce in samo ce gre za vecmap if self.args.map_method == 'vecmap': # map feat_embs with vecmap, actually self.mapper defined in class init feat_embs = self.mapper.map_batch(feat_embs) mask = words.ne(self.WORD.pad_index) # ignore the first token of each sentence mask[:, 0] = 0 feats0 = torch.zeros(words.shape+(1024,)) # words.clone() feats1 = torch.zeros(words.shape+(1024,)) feats2 = torch.zeros(words.shape+(1024,)) # words get ignored, all input comes from feats - 3 elmo layers # still inputting words due to reasons(tm) #feats0 = feats0.unsqueeze(-1) #feats0 = feats0.expand(words.shape+(1024,)) for sentence in range(len(feat_embs)): for token in range(len(feat_embs[sentence][1])): feats0[sentence][token] = torch.Tensor(feat_embs[sentence][0][token]) feats1[sentence][token] = torch.Tensor(feat_embs[sentence][1][token]) feats2[sentence][token] = torch.Tensor(feat_embs[sentence][2][token]) feats = torch.cat((feats0, feats1, feats2), -1) if str(self.args.device) == '-1': feats = feats.to('cpu') else: feats = feats.to('cuda:'+str(self.args.device)) #TODO: fix to allow cpu or gpu s_arc, s_rel = self.model(words, feats) #INFO: here is the data input, y = model(x) loss = self.model.loss(s_arc, s_rel, arcs, rels, mask, self.args.partial) loss.backward() nn.utils.clip_grad_norm_(self.model.parameters(), self.args.clip) self.optimizer.step() self.scheduler.step() arc_preds, rel_preds = self.model.decode(s_arc, s_rel, mask) if self.args.partial: mask &= arcs.ge(0) # ignore all punctuation if not specified if not self.args.punct: mask &= words.unsqueeze(-1).ne(self.puncts).all(-1) metric(arc_preds, rel_preds, arcs, rels, mask) bar.set_postfix_str(f"lr: {self.scheduler.get_last_lr()[0]:.4e} - loss: {loss:.4f} - {metric}")
def _evaluate(self, loader): print("called _evaluate function") print(self.mapper) self.model.eval() total_loss, metric = 0, AttachmentMetric() for words, feats, arcs, rels in loader: if self.elmo: feat_embs0 = self.elmo.embed_batch(feats) else: feat_embs0 = self.efml.sents2elmo(feats, output_layer=-2) if self.mapper: # map feat_embs with self.mapper defined in class init feat_embs = self.mapper.map_batch(feat_embs0) else: feat_embs = feat_embs0 mask = words.ne(self.WORD.pad_index) # ignore the first token of each sentence mask[:, 0] = 0 feats0 = torch.zeros(words.shape+(1024,)) feats1 = torch.zeros(words.shape+(1024,)) feats2 = torch.zeros(words.shape+(1024,)) for sentence in range(len(feat_embs)): for token in range(len(feat_embs[sentence][1])): feats0[sentence][token] = torch.Tensor(feat_embs[sentence][0][token]) feats1[sentence][token] = torch.Tensor(feat_embs[sentence][1][token]) feats2[sentence][token] = torch.Tensor(feat_embs[sentence][2][token]) feats = torch.cat((feats0, feats1, feats2), -1) if str(self.args.device) == '-1': feats = feats.to('cpu') else: feats = feats.to('cuda:'+str(self.args.device)) s_arc, s_rel = self.model(words, feats) loss = self.model.loss(s_arc, s_rel, arcs, rels, mask, self.args.partial) arc_preds, rel_preds = self.model.decode(s_arc, s_rel, mask, self.args.tree, self.args.proj) if self.args.partial: mask &= arcs.ge(0) # ignore all punctuation if not specified if not self.args.punct: mask &= words.unsqueeze(-1).ne(self.puncts).all(-1) total_loss += loss.item() metric(arc_preds, rel_preds, arcs, rels, mask) total_loss /= len(loader) return total_loss, metric
def _evaluate(self, loader): self.model.eval() total_loss, metric = 0, AttachmentMetric() for words, texts, *feats, arcs, sibs, rels in loader: word_mask = words.ne(self.args.pad_index) mask = word_mask if len(words.shape) < 3 else word_mask.any(-1) # ignore the first token of each sentence mask[:, 0] = 0 s_arc, s_sib, s_rel = self.model(words, feats) loss, s_arc = self.model.loss(s_arc, s_sib, s_rel, arcs, sibs, rels, mask, self.args.mbr, self.args.partial) arc_preds, rel_preds = self.model.decode(s_arc, s_sib, s_rel, mask, self.args.tree, self.args.mbr, self.args.proj) if self.args.partial: mask &= arcs.ge(0) # ignore all punctuation if not specified if not self.args.punct: mask.masked_scatter_(mask, ~mask.new_tensor([ispunct(w) for s in texts for w in s])) total_loss += loss.item() metric(arc_preds, rel_preds, arcs, rels, mask) total_loss /= len(loader) return total_loss, metric