def __init__(self): with Timer() as self.model_load_time: from iwnlp.iwnlp_wrapper import IWNLPWrapper from stts2upos import conv_table data_loc = "/opt/iwnlp/IWNLP.Lemmatizer_20181001.json" self.lemmatizer = IWNLPWrapper(lemmatizer_path=data_loc) def myprocessor(myinput): mydoc = string2doc(myinput) for sent in mydoc: for tok in sent: try: matching_lemmas = self.lemmatizer.lemmatize( tok.word, conv_table.get(tok.xpos)) if matching_lemmas is None: tok.lemma = "_" # elif len(matching_lemmas) > 1: # print("lots o lemmas!", matching_lemmas) else: # unclear how to select best alternative # just use first item in list tok.lemma = matching_lemmas[0] except ValueError: tok.lemma = "_" # don't repeat gold pos in output tok.hide_fields(HIDDEN_FIELDS) return mydoc self.processor = myprocessor
def __init__(self, lemmatizer_path, nlp): self.lemmatizer = IWNLPWrapper(lemmatizer_path=lemmatizer_path) self.stringstore = nlp.vocab.strings # self.matcher = PhraseMatcher(nlp.vocab) Token.set_extension('iwnlp_lemmas', getter=self.lemmatize, force=True) self.lookup = { ('fast', ADV): 'fast', }
def __init__(self, lemmatizer_path, use_plain_lemmatization=False, ignore_case=False): self.lemmatizer = IWNLPWrapper(lemmatizer_path=lemmatizer_path) self.use_plain_lemmatization = use_plain_lemmatization self.ignore_case = ignore_case Token.set_extension('iwnlp_lemmas', getter=self.get_lemmas, force=True)
def __init__(self): self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) self.lemmatizer = IWNLPWrapper( lemmatizer_path='data/IWNLP/IWNLP.Lemmatizer_20170501.json') self.sentiws = SentiWSWrapper(sentiws_path='data/sentiws') self.logger.debug('Loading Spacy model') self.nlp = spacy.load('de') self.logger.debug('Spacy model loaded')
def preprocess(self): tokenizedTweets_writer = open( './daten/tokenized_tweets_normalized.txt', 'w') preprocTweets_writer = open( './daten/preprocessed_tweets_normalized.txt', 'w') pp = Pipeline(self.this_file, "./autosarkasmus/rsrc/de-tiger.map") tweets_tkn, tweets_proc, labels = pp.process() assert (len(tweets_tkn) == len(tweets_proc) == len(labels)) # filter stopwords + normalize tokens lemmatizer = IWNLPWrapper( lemmatizer_path='daten/IWNLP.Lemmatizer_20170501.json') lemmatized_tokens = [] for x in range(len(tweets_tkn)): tweet = [] for token in tweets_tkn[x]: if token.lower() in stopwords.words('german'): continue try: lemma = lemmatizer.lemmatize_plain(token, ignore_case=True) if (lemma): tweet.append(lemma[0]) else: tweet.append(token) except Exception as e: print(e) lemmatized_tokens.append(tweet) assert (len(lemmatized_tokens) == len(tweets_proc) == len(labels)) # write preprocessing results to file for x in range(len(lemmatized_tokens)): t_tweet = (" ").join(lemmatized_tokens[x]) p_tweet = (" ").join( [str(x) + "/" + str(y) for x, y in tweets_proc[x]]) label = labels[x] tokenizedTweets_writer.write(t_tweet + "\t" + label + "\n") preprocTweets_writer.write(p_tweet + "\t" + label + "\n")
def load(cls, lemmatizer_path): lemmatizer = IWNLPWrapper(lemmatizer_path=lemmatizer_path) return cls(lemmatizer)
class Preprocess: # zur Lemmatisierung im Deutschen nlp = spacy.load('de') # IWNLP German Lemmatizations: dirname = os.path.dirname(__file__) iwnlp_file = os.path.join(dirname, 'data/IWNLP.Lemmatizer_20181001.json') #iwnlp = spaCyIWNLP(lemmatizer_path='data/IWNLP.Lemmatizer_20181001.json', ignore_case=True) lemmatizer = IWNLPWrapper(lemmatizer_path=iwnlp_file) #add custom tokenizer nlp.tokenizer = custom_tokenizer(nlp) ''' try: # add pipes nlp.add_pipe(iwnlp) # nlp.add_pipe(__set_custom_boundaries, before='parser') except Exception: pass ''' stopwords_to_remove_from_default_set = [ "schlecht", "mensch", "menschen", "beispiel", "gott", "jahr", "jahre", "jahren", "nicht", "uhr" ] for stopword in stopwords_to_remove_from_default_set: nlp.vocab[stopword].is_stop = False #Spacy Token Tags, which will be removed by preprocessing tags_to_remove = [ '$(', '$,', '$.', 'APPR', 'APPO', 'APPRART', 'APZR', 'ART', 'ITJ', 'KOKOM', 'KON', 'KOUI', 'KOUS', # 'CARD', 'PDS', 'PAV', 'PROAV', 'PDAT', 'PIAT', 'PIDAT', 'PIS', 'PPER', 'PPOSAT', 'PPOSS', 'PRELAT', 'PRELS', 'PRF', 'PTKA', # 'PTKANT', 'PTKVZ', 'PTKZU', 'PWAT', 'PWAV', 'PWS', 'TRUNC', 'XY', 'SP', 'WRP' ] def __init__(self, text, split_in_sentences=True, with_pos=False): ''' :param text: input text :param split_in_sentences: split text in sentences --> sub-arrays for sentences in Preprocess-result :param with_pos: true: give tripel with (<startpos in orig-text>, <endpos in origtext>, token), else only tokens ''' self.text = text self.nlp_text = self.nlp(text) self.maintain_indeces = [] self.noun_chunks = self.get_noun_chunks(cleaned=True, flattened=True) self.maintain_indeces.extend(index for index in self.noun_chunks if index not in self.maintain_indeces) self.named_entities = self.get_named_entities(flattened=True) self.maintain_indeces.extend(index for index in self.named_entities if index not in self.maintain_indeces) self.maintain_indeces.sort() self.preprocessed = self.preprocess(sentence_split=split_in_sentences, with_pos=with_pos) def __get_lemma(self, token): ''' take lemma of IWNLP, if given, else spacy lemma :param token: spacy-token :return: lemmatization ''' #lemma_iwnlp_list = token._.iwnlp_lemmas lemma_iwnlp_list = self.lemmatizer.lemmatize_plain(token.text, ignore_case=False) if lemma_iwnlp_list: lemma_iwnlp = lemma_iwnlp_list[0] #print(token, ":::", lemma_iwnlp_list[0]) return lemma_iwnlp return token.lemma_ def get_named_entities(self, only_indeces=True, flattened=False): ''' return array of named entities (PER: Person, LOC: Location, ORG: Named corporate, governmental, or other organizational entity, MISC: Miscellaneous entities, e.g. events, nationalities, products or works of art) :param only_indeces: :param flattened: returns only 1d array, else related entities are in sup-arrays :return: array with named entities ''' if flattened: named_ents = [ word.i if only_indeces else (word.i, word, ents.label_) for ents in self.nlp_text.ents for word in ents ] else: named_ents = [[ word.i if only_indeces else (word.i, word, ents.label_) for word in ents ] for ents in self.nlp_text.ents] return named_ents def get_noun_chunks(self, only_indices=True, cleaned=True, flattened=False): ''' return array of noun_chunks/noun_phrases of the Text object :param only_indices: :param cleaned: noun phrases without stopword, punctuation :param flattened: returns only 1d array, else related phrases are in sup-arrays :return: array with noun-phrases ''' # noun_words = [(word.i, word) for ent in text.noun_chunks for word in ent] # noun_words = [[(word.i, word) for word in ent] for ent in text.noun_chunks] if flattened: if cleaned: noun_words = [ word.i if only_indices else (word.i, word) for ent in self.nlp_text.noun_chunks for word in ent if self.__is_valid_token(word) ] else: noun_words = [ word.i if only_indices else (word.i, word) for ent in self.nlp_text.noun_chunks for word in ent ] else: if cleaned: noun_words = [[ word.i if only_indices else (word.i, word) for word in ent if self.__is_valid_token(word) ] for ent in self.nlp_text.noun_chunks] else: noun_words = [[ word.i if only_indices else (word.i, word) for word in ent ] for ent in self.nlp_text.noun_chunks] return noun_words def __is_valid_token(self, token): ''' checks if token is valid: no stopword, punctuation oder whitespace :param token: spacy-token :return: bool ''' # nlp(token.lower_)[0] wegen spacy bug --> z.B. "Der" würde nicht als stopwort erkannt werden, "der" aber schon if not self.nlp( token.lower_ )[0].is_stop and not token.is_punct and not token.is_space: return True return False def __tokenize_words(self, doc, with_pos=False): ''' tokenizes text and removes unimportant tokens :param doc: input spacy doc :param with_pos: true: give tripel with (<startpos in orig-text>, <endpos in origtext>, token), else only tokens :return: 1d array of tokens ''' tokenized_text = [ (token.idx, token.idx + len(token), self.__get_lemma(token).lower()) if with_pos else self.__get_lemma(token).lower() for token in doc if self.__is_valid_token(token) and not token.tag_ in self.tags_to_remove or token.i in self.maintain_indeces ] return tokenized_text def __tokenize_to_list_sentences(self, with_pos=False): ''' tokenizes text and removes unimportant tokens, split by sentences :param with_pos: true: give tripel with (<startpos in orig-text>, <endpos in origtext>, token), else only tokens :return: 2d array of tokens in sub-arrays (sentences) ''' filtered_text = [] for sentence in self.nlp_text.sents: filtered_sentence = self.__tokenize_words(sentence, with_pos=with_pos) filtered_text.append(filtered_sentence) return filtered_text def preprocess(self, sentence_split=True, with_pos=False): ''' preprocess text. removes unimportant tokens :param sentence_split: split by sentences :param with_pos: true: give tripel with (<startpos in orig-text>, <endpos in origtext>, token), else only tokens :return: 1d or 2d array with preprocessed text ''' if sentence_split: preprocessed_text = self.__tokenize_to_list_sentences( with_pos=with_pos) else: preprocessed_text = self.__tokenize_words(self.nlp_text, with_pos=with_pos) return preprocessed_text
def setUpClass(self): self.iwnlp = IWNLPWrapper(lemmatizer_path='data/IWNLP.Lemmatizer_20170501.json')
from iwnlp.iwnlp_wrapper import IWNLPWrapper output = 'output/Baselist.txt' path = "Baselist.txt" lemmatizer = IWNLPWrapper(lemmatizer_path='IWNLP.Lemmatizer_20170501.json') with open(path, 'r') as read_file: data = read_file.read().splitlines() tokens = [] lemmatized = [] tags = [] for i in data: j = i.split('|') tokens.append(j[0].lower()) tags.append(j[1]) for token, tag in zip(tokens, tags): lemma = lemmatizer.lemmatize(token, pos_universal_google=tag) lemmatized.append(lemma) for token, lemma in zip(tokens, lemmatized): '' # print(token, lemma) with open(output, 'w') as write_file: for i, lemma in enumerate(lemmatized): if (lemmatized[i] == None): write_file.write(tokens[i] + '|' + tags[i] + "\n") else:
def __init__(self, lang): self.lang = lang self.nlp = nlp = spacy.load(lang) current_dir = os.path.dirname(__file__) self.iwnlp = IWNLPWrapper(lemmatizer_path=current_dir + '/../resources/IWNLP.Lemmatizer_20170501.json')