def remove_contradicting_pairs(all_pairs, conjugated_words, disable_tqdm=False, verbose=True): conj_words = {word: OrderedSet() for word in conjugated_words} # Group all tuples by their first word for pair in all_pairs: word1 = pair[0] conj_words[word1].add(pair) # Make sure each conjugated word is only paired with one form of another word for word, pairs in tqdm(conj_words.items(), disable=disable_tqdm): pairs_cleaned = OrderedSet() paired_words = OrderedSet() for pair in pairs: word2 = pair[1] word2_lemma = lemma(word2) if word2_lemma not in paired_words: paired_words.add(word2_lemma) pairs_cleaned.add(pair) else: # If this word stem already appears here it must be removed. Otherwise syntactically different forms # of the same stem will appear with the same other word and hence be mapped closely together # in attract-repel if verbose: print("Removed:", pair, "From:", pairs) conj_words[word] = pairs_cleaned all_pairs_cleaned = OrderedSet() for pairs in conj_words.values(): all_pairs_cleaned.update(pairs) return all_pairs_cleaned
def evaluate(self, node): words = node.words() A = Analyzer.instance() corp = A.get(corpus="TIGER") sents_count = len(node.sents(tokenizer=corp.sent_tokenizer())) tagger = corp.tagger(True) tagged_words = tagger.tag(words) count = 0 if len(tagged_words) > 0: for w in tagged_words: if w[1] and w[1].startswith("V"): lemm = lemma(w[0]) if lemm in self.VERBS: count += 1 return float(count) / sents_count return 0.0
def evaluate(self, node): words = node.words() words_no_no_words = [w for w in words if w not in NO_WORDS] A = Analyzer.instance() corp = A.get(corpus="TIGER") tagger = corp.tagger(True) tagged_words = tagger.tag(words) unique_words = set() if len(tagged_words) > 0 and len(words_no_no_words) > 0: for w in tagged_words: if w[0] not in NO_WORDS: if w[1] and w[1].startswith("V"): lemm = lemma(w[0]) unique_words.add(lemm) else: unique_words.add(w[0]) return float(len(unique_words)) / len(words_no_no_words) return 0.0
def _getLemma(word, language): import pattern.en as pattern_en # @UnresolvedImport import pattern.es as pattern_es # @UnresolvedImport import pattern.fr as pattern_fr # @UnresolvedImport import pattern.de as pattern_de # @UnresolvedImport import pattern.it as pattern_it # @UnresolvedImport if language == "es": return pattern_es.lemma(word) elif language == "en": return pattern_en.lemma(word) elif language == "it": return pattern_it.lemma(word) elif language == "fr": return pattern_fr.lemma(word) elif language == "de": return pattern_de.lemma(word) else: return pattern_en.lemma(word)
# Author: Devon Fritz # Date: 30.5.15 # Stems the lexicon of german words import sys from pattern.de import lemma reload(sys) sys.setdefaultencoding('utf-8') input_file = sys.argv[1] with open(input_file) as f: for line in f: print lemma(line.split()[0].strip()) + (' ' + line.split()[1] if len(line.split()) > 1 else '')
# Author: Devon Fritz # Date: 30.5.15 # Stems the lexicon of german words import sys from pattern.de import lemma reload(sys) sys.setdefaultencoding('utf-8') input_file = sys.argv[1] with open(input_file) as f: for line in f: print lemma(line.split()[0].strip()) + (' ' + line.split()[1] if len(line.split()) > 1 else '')
# Author: Devon Fritz # Date: 30.5.15 # Stems the lexicon of german words import sys from pattern.de import lemma, lexeme reload(sys) sys.setdefaultencoding('utf-8') input_file = sys.argv[1] with open(input_file) as f: for line in f: print lemma(line.strip()) for l in lexeme(line.strip()): print l
import sys import re import codecs # not used, but possibly interesting http://www.nltk.org/ # http://www.clips.ua.ac.be/pages/pattern-de from pattern.de import lemma, tag, predicative, singularize # possible parts of speech: # PRP$, FW, VBN, WDT, JJ, WP, DT, RP, NN, TO, PRP, # RB, NNS, NNP, VB, WRB, CC, LS, CD, IN, MD, UH part_of_speech_command = { 'PRP$': lambda word: predicative(word), # pronomina 'VBN': lambda word: lemma(word), # verba 'DT': lambda word: predicative(word), # pronomina 'VB': lambda word: lemma(word), # verba 'NN': lambda word: singularize(word), # nomina 'JJ': lambda word: predicative(word) # preposice } pattern_word = re.compile('[a-zA-Z]') pattern_punctuation = re.compile(ur'[—\-|«»…–<>]') def transform(tagword): word = tagword[0] part = tagword[1] # if part == 'VBN': # print tagword