def __call__(self, text): tokens = list(self.tokenizer(text)) tokens_lemmas = lemmatize([t.text for t in tokens], self._morph) tags = self.network.predict_for_token_batch([tokens_lemmas])[0] previous_tag = null_tag = 'O' previous_tokens = [] for token, current_tag in zip( itertools.chain(tokens, [None]), itertools.chain(tags, [null_tag]) ): if current_tag.startswith('I'): previous_tokens.append(token) elif previous_tag != null_tag: yield Match( previous_tokens, Span( previous_tokens[0].span[0], previous_tokens[-1].span[1], ), previous_tag[-3:] ) if current_tag.startswith('B'): previous_tokens = [token] previous_tag = current_tag
def __call__(self, text): tokens = list(self.tokenizer(text)) tokens_lemmas = lemmatize([t.text for t in tokens], self._morph) tags = self.network.predict_for_token_batch([tokens_lemmas])[0] previous_tag = null_tag = 'O' previous_tokens = [] for token, current_tag in zip( itertools.chain(tokens, [None]), itertools.chain(tags, [null_tag]) ): if current_tag.startswith('I'): previous_tokens.append(token) elif previous_tag != null_tag: yield Match( previous_tokens, Span( previous_tokens[0].span[0], previous_tokens[-1].span[1], ), previous_tag[-3:] ) if current_tag.startswith('B'): previous_tokens = [token] previous_tag = current_tag
def print_predict(sentence): # Split sentence into tokens tokens = tokenize(sentence) # Lemmatize every token # Example: был -> быть, его -> он tokens_lemmas = lemmatize(tokens) tags = network.predict_for_token_batch([tokens_lemmas])[0] for token, tag in zip(tokens, tags): print(token, tag)
def print_predict(sentence, network, f=sys.stdout, threshold=0.2): # Split sentence into tokens tokens = tokenize(sentence) # Lemmatize every token tokens_lemmas = lemmatize(tokens) tags, logits = network.predict_for_token_batch([tokens_lemmas]) tags, logits = tags[0], logits[0] o_idx = network.corpus.tag_dict.toks2idxs(['O']) predicted_tags = [] last_number = None last_tk = None for token, tag, l in zip(tokens, tags, logits): if is_number(token.lower()): last_number = normalize_num(token) second_best = np.argsort(l)[-2] third_best = np.argsort(l)[-3] if tag == 'O': ratio = l[second_best] / l[third_best] #if ratio * l[second_best] > .2 and token not in '.?,\':!': if ratio * l[ second_best] > threshold and token not in '.?,\':!' and token not in stopwords.words( 'english'): tag = network.corpus.tag_dict.idxs2toks([second_best])[0] elif tag.startswith('B') and (token in stopwords.words('english') or token in '.?,\':!'): tag = 'O' # print(token, tag, file=f) if tag.startswith('B') and token not in stopwords.words( 'english') and token not in '.?,\':!': if 'calendric' in tag and 'night' in token and last_number is not None: predicted_tags.append(('nights', last_number)) last_number = None elif 'people' in tag and last_number is not None: predicted_tags.append(('people', last_number)) last_number = None elif 'performers' in tag and last_number is not None: predicted_tags.append(('stars', last_number)) last_number = None elif 'gpe' in tag: if last_tk == 'to': predicted_tags.append(('toloc.city_name', token)) else: predicted_tags.append(('fromloc.city_name', token)) else: predicted_tags.append((tag[2:], token)) last_tk = token return predicted_tags
def _preprocess_task(self, task): # tokens = tokenize(task) tokens = task.split(' ') tokens_lemmas = lemmatize(tokens) return tokens_lemmas
def _preprocess_humaninput(self, task): tokens = nltk.tokenize.wordpunct_tokenize(task) tokens_lemmas = lemmatize(tokens) return tokens_lemmas