def get_features(ptree: nltk.ParentedTree, dtree: List[DepRel], indices, sense, offset): features_sentence = [] for i, drel in enumerate(dtree): if drel.rel.lower() == 'root': mv_position = i break else: mv_position = 0 main_verb = lemmatizer.lemmatize(ptree.pos()[mv_position][0]) for i, (word, tag) in enumerate(ptree.pos()): tree_pos = ptree.treeposition_spanning_leaves(i, i + 1)[:-2] chain = [ptree[tree_pos[:i + 1]].label() for i in range(len(tree_pos))] chain = ['S' if c == 'SBAR' else c for c in chain] if len(chain) > 0: chain = "-".join(get_compressed_chain(chain)) stem = stemmer.stem(word).lower() features_word = { 'idx': offset + i, 'BOS': i == 0, 'word': word.lower(), 'pos': tag, 'lemma': lemmatizer.lemmatize(word).lower(), 'stem': stem.lower(), 'chain': chain, 'conn': sense.split('.')[0] if offset + i in indices else "", 'inflection': word[len(stem):], 'is_main_verb': i == mv_position, 'main_verb': main_verb.lower() } features_sentence.append(features_word) return features_sentence
def get_features(ptree: nltk.ParentedTree, conn_idxs): leave_list = ptree.leaves() lca_loc = ptree.treeposition_spanning_leaves(conn_idxs[0], conn_idxs[-1] + 1)[:-1] self_category = ptree[lca_loc].label() parent_category = ptree[lca_loc].parent().label() if lca_loc else self_category left_sibling = get_sibling_label(ptree[lca_loc], 'left') right_sibling = get_sibling_label(ptree[lca_loc], 'right') labels = {n.label() for n in ptree.subtrees(lambda t: t.height() > 2)} bool_vp = 'VP' in labels bool_trace = 'T' in labels c = ' '.join(leave_list[conn_idxs[0]:conn_idxs[-1] + 1]).lower() prev, prev_conn, prev_pos, prev_pos_conn_pos = get_pos_features(ptree, conn_idxs, c, -1) next, next_conn, next_pos, next_pos_conn_pos = get_pos_features(ptree, conn_idxs, c, 1) prev = lemmatizer.lemmatize(prev) next = lemmatizer.lemmatize(next) r2l = [ptree[lca_loc[:i + 1]].label() for i in range(len(lca_loc))] r2lcomp = get_compressed_chain(r2l) feat = {'connective': c, 'connectivePOS': self_category, 'prevWord': prev, 'prevPOSTag': prev_conn, 'prevPOS+cPOS': prev_pos_conn_pos, 'nextWord': next, 'nextPOSTag': next_pos, 'cPOS+nextPOS': next_pos_conn_pos, 'root2LeafCompressed': ','.join(r2lcomp), 'root2Leaf': ','.join(r2l), 'left_sibling': left_sibling, 'right_sibling': right_sibling, 'parentCategory': parent_category, 'boolVP': bool_vp, 'boolTrace': bool_trace} return feat