def nonlinearity(tensor, xs, ys): assert len(xs) == len(ys) output = torch.ones_like(tensor, device=tensor.device) * ys[0] for (x1, x2), (y1, y2) in zip(windowed(xs, 2), windowed(ys, 2)): output += (y2 - y1) / (x2 - x1) * (torch.clamp(tensor, x1, x2) - x1) return output
def nonlinearity(tensor, xs, ys): # TODO: This could probably be sped up quite a bit with torch.search_sorted assert len(xs) == len(ys) output = torch.ones_like(tensor, device=tensor.device) * ys[0] for (x1, x2), (y1, y2) in zip(windowed(xs, 2), windowed(ys, 2)): output += (y2 - y1) / (x2 - x1) * (torch.clamp(tensor, x1, x2) - x1) return output
def _words_per_seg(self, lengths, labels): """ Count words included in each segment """ # Get indexes of boundary segmentations (where 1s are) indexes = self._boundary_ids(labels) # How many words are in each sentence seg_words = [sum(lengths[i1:i2]) for i1, i2 in windowed(indexes, 2)] # Happens when zero segmentations predicted if not seg_words: seg_words = [sum(lengths)] return seg_words
def compute_idx_spans(sentences, L=10): """ Compute span indexes for all possible spans up to length L in each sentence """ idx_spans, shift = [], 0 for sent in sentences: sent_spans = flatten([ windowed(range(shift, len(sent) + shift), length) for length in range(1, L) ]) idx_spans.extend(sent_spans) shift += len(sent) return idx_spans
def reravel(self): """ Group sentences back into subsections (Used in Hearst baseline) """ intervals = windowed(self.indexes, 2) reraveled = [self.sents[i:j] for i, j in windowed(self.indexes, 2)] return reraveled
def spans_in_sentence(sent, length,shift): return windowed(range(shift, len(sent)+shift), length)
def _indexes(aligned): """ Return tuples of (start, stop) indexes for documents """ return windowed([ i for i, (title, _) in enumerate(aligned) if title == 'TOP-LEVEL SEGMENT' ] + [len(aligned)], 2)
def ngrams(self, n=1): for ng in windowed(self.tokens, n): yield '_'.join(ng)
def compute_idx_spans(tokens, L=10): """ Compute all possible token spans """ return flatten( [windowed(range(len(tokens)), length) for length in range(1, L)])
def pair(spans): return windowed(spans, 2)