def create_new_corpus(data_dict, corpus_vol, **kwargs): new_corpus = Corpus([]) sem_nums = kwargs['sem_nums'] intents = data_dict.keys() if not corpus_vol: return elif sem_nums > len(intents): return else: for i in range(corpus_vol): intent_sam = set() while len(intent_sam) < sem_nums: intent_sam.add(random.choice(list(intents))) spanset = SpanSet() sentences = [] start_position = 0 for intent in list(intent_sam): if intent == 'noise': txt = random.choice(list(data_dict[intent])) sentences.append(txt) start_position += len(txt) else: txt = random.choice(list(data_dict[intent])) sentences.append(txt) spanset.append( Span(start=start_position, end=start_position + len(txt), entity=intent)) start_position += len(txt) doc = Document(text=''.join(sentences), label='|'.join(intent_sam), span_set=spanset) new_corpus.append(doc) return new_corpus
def test_eq_(): a = SpanSet() a.append(Span(1, 2, 'entity')) a.append(Span(2, 3, 'entity')) b = SpanSet() b.append(Span(1, 2, 'entity')) b.append(Span(2, 3, 'entity')) assert a == b c = SpanSet() # empty SpanSet assert a != c d = SpanSet() # same with `a` but different span order d.append(Span(2, 3, 'entity')) d.append(Span(1, 2, 'entity')) assert a == d e = SpanSet() # same with `a` but different span order e.append(Span(0, 1, 'entity')) e.append(Span(1, 2, 'entity')) assert a != e
def _turn_training_data_to_offset(training_data): from tokenizer_tools.tagset.offset.sequence import Sequence from tokenizer_tools.tagset.offset.span import Span from tokenizer_tools.tagset.offset.span_set import SpanSet for example in training_data.training_examples: span_set = SpanSet() text = [i for i in example.text] # need to be str list (not str) intent = example.get("intent") for ent in example.get("entities", []): start, end, entity = ent["start"], ent["end"], ent["entity"] span_set.append(Span(start, end, entity)) seq = Sequence(text, span_set, label=intent) yield seq
def test_check_overlap(): span_set = SpanSet() span_set.append(Span(1, 2, 'entity')) span_set.append(Span(2, 3, 'entity')) assert span_set.check_overlap()[0] == True span_set = SpanSet() span_set.append(Span(1, 2, 'entity')) span_set.append(Span(4, 6, 'entity')) assert span_set.check_overlap()[0] == True span_set = SpanSet() span_set.append(Span(1, 4, 'entity')) span_set.append(Span(2, 3, 'entity')) check_result = span_set.check_overlap() assert check_result[0] == False assert check_result[1] == [(Span(1, 4, 'entity'), Span(2, 3, 'entity'))]
def test_check_match(): span_set = SpanSet() span_set.append(Span(1, 2, 'entity', '春')) span_set.append(Span(2, 3, 'entity', '秋')) assert span_set.check_match('赛春秋')[0] == True span_set = SpanSet() span_set.append(Span(1, 2, 'entity', '春')) span_set.append(Span(4, 6, 'entity', '秋天')) assert span_set.check_match('赛春秋赛秋天')[0] == True span_set = SpanSet() span_set.append(Span(1, 4, 'entity', '赛春秋')) span_set.append(Span(2, 3, 'entity', '春')) assert span_set.check_match('赛赛春秋')[0] == True span_set = SpanSet() span_set.append(Span(1, 4, 'entity', '赛春秋')) span_set.append(Span(2, 3, 'entity', '春')) check_result = span_set.check_match('不不不不') assert check_result[0] == False assert check_result[1] == [ Span(1, 4, 'entity', '赛春秋'), Span(2, 3, 'entity', '春') ]