def preprocess(text, NUM_DOCS, num_preprocessed, stemming): global i if i == 0: i = num_preprocessed i += 1 result = [] stemmer = ItalianStemmer() if i % 20 == 0: print(f"\t{i} out of {NUM_DOCS+num_preprocessed} documents preprocessed") nlp = Italian() t0 = text.split("Lingua processuale")[0].split("Sentenza")[-1] t1 = "".join(t0) t1 = re.sub(r"’|'|«|»|\d{1,4}\/\d{1,4}\/(cee|ce)|\d+|---\|*|^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$", " ", t1, flags=re.IGNORECASE) # print(t1) doc = nlp(t1) for token in doc: if token.text.lower() not in it_stopwords and not token.is_punct | token.is_space and len(token) > 3: assert token.lang_ == "it" if stemming: result.append(stemmer.stem(word=token.text)) else: result.append(token.lemma_.lower()) if "'" in result[-1] or "’" in result[-1]: raise Exception(f"Detected_ {token.lemma_}") return result
def init_resources(self): self.punctuation_pattern = re.compile("|".join(PUNCTUATION)) self.stemmer = None stopwords_path = os.path.join( os.path.dirname(assistant_dialog_skill_analysis.__file__), "resources", self.language_code, "stopwords", ) if self.language_code == "en": from spacy.lang.en import English self.tokenizer = Tokenizer(English().vocab) self.stemmer = SnowballStemmer(language="english") self.stop_words = self.load_stop_words(stopwords_path) elif self.language_code == "fr": from spacy.lang.fr import French self.tokenizer = Tokenizer(French().vocab) self.stemmer = SnowballStemmer(language="french") self.stop_words = self.load_stop_words(stopwords_path) elif self.language_code == "de": from spacy.lang.de import German self.tokenizer = Tokenizer(German().vocab) self.stemmer = SnowballStemmer(language="german") self.stop_words = self.load_stop_words(stopwords_path) elif self.language_code == "it": from spacy.lang.it import Italian self.tokenizer = Tokenizer(Italian().vocab) self.stemmer = SnowballStemmer(language="italian") self.stop_words = self.load_stop_words(stopwords_path) elif self.language_code == "cs": from spacy.lang.cs import Czech self.tokenizer = Tokenizer(Czech().vocab) self.stop_words = self.load_stop_words(stopwords_path) elif self.language_code == "pt": from spacy.lang.pt import Portuguese self.tokenizer = Tokenizer(Portuguese().vocab) self.stemmer = SnowballStemmer(language="portuguese") self.stop_words = self.load_stop_words(stopwords_path) elif self.language_code == "es": from spacy.lang.es import Spanish self.tokenizer = Tokenizer(Spanish().vocab) self.stemmer = SnowballStemmer(language="spanish") self.stop_words = self.load_stop_words(stopwords_path) else: raise Exception("language code %s is not supported", self.language_code)
def get_tokenizers(self, lang): os.environ['TOKENIZERS_PARALLELISM'] = "True" if lang == 'de': spacy = German() bert = "deepset/gbert-base" elif lang == 'fr': spacy = French() bert = "camembert/camembert-base-ccnet" elif lang == 'it': spacy = Italian() bert = "dbmdz/bert-base-italian-cased" else: raise ValueError( f"Please choose one of the following languages: {self.languages}" ) return spacy.tokenizer, AutoTokenizer.from_pretrained(bert)
def lang_change(language): if language == 'en': from spacy.lang.en import English from spacy.lang.en.stop_words import STOP_WORDS parser = English() file = "\config_files\config_spacy_en.yaml" configfile_path = os.getcwd() + file elif language == 'de': from spacy.lang.de import German from spacy.lang.de.stop_words import STOP_WORDS parser = German() file = "\config_files\config_spacy_de.yaml" configfile_path = os.getcwd() + file elif language == 'es': from spacy.lang.es import Spanish from spacy.lang.es.stop_words import STOP_WORDS parser = Spanish() file = "\config_files\config_spacy_es.yaml" configfile_path = os.getcwd() + file elif language == 'pt': from spacy.lang.pt import Portuguese from spacy.lang.pt.stop_words import STOP_WORDS parser = Portuguese() file = "\config_files\config_spacy_pt.yaml" configfile_path = os.getcwd() + file elif language == 'fr': from spacy.lang.fr import French from spacy.lang.fr.stop_words import STOP_WORDS parser = French() file = "\config_files\config_spacy_fr.yaml" configfile_path = os.getcwd() + file elif language == 'it': from spacy.lang.it import Italian from spacy.lang.it.stop_words import STOP_WORDS parser = Italian() file = "\config_files\config_spacy_it.yaml" configfile_path = os.getcwd() + file elif language == 'nl': from spacy.lang.nl import Dutch from spacy.lang.nl.stop_words import STOP_WORDS parser = Dutch() file = "\config_files\config_spacy_nl.yaml" configfile_path = os.getcwd() + file return parser, STOP_WORDS, configfile_path
text = word_tokenize(open("data/converted/" + doc, "r", encoding='utf-8').read(), language='italian') average += len(text) print(average / i, i) av1 = 0 av2 = 0 i = 0 for dic in os.listdir("data/.preprocessed"): year = pickle.load(open("data/.preprocessed/" + dic, "rb")) i += len(year) print(len(year)) for doc in year: if dic[0] == 's': av1 += len(doc) else: av2 += len(doc) print(av1 / i, av2 / i, i) nlp = Italian() doc = nlp(open("data/converted/61999CJ0001.txt", "r", encoding='utf-8').read()) tok = [] for t in doc: tok.append(t.lemma_.lower()) dictionary = Dictionary([tok])
def test_issue2179(): """Test that spurious 'extra_labels' aren't created when initializing NER.""" nlp = Italian() ner = nlp.create_pipe("ner") ner.add_label("CITIZENSHIP") nlp.add_pipe(ner) nlp.begin_training() nlp2 = Italian() nlp2.add_pipe(nlp2.create_pipe("ner")) nlp2.from_bytes(nlp.to_bytes()) assert "extra_labels" not in nlp2.get_pipe("ner").cfg assert nlp2.get_pipe("ner").labels == ("CITIZENSHIP", )
def test_issue2482(): """Test we can serialize and deserialize a blank NER or parser model.""" nlp = Italian() nlp.add_pipe(nlp.create_pipe("ner")) b = nlp.to_bytes() Italian().from_bytes(b)
from datasketch import MinHash, MinHashLSH from nltk.corpus import stopwords from nltk.stem import WordNetLemmatizer from spacy.lang.en import English from spacy.lang.it import Italian import whoosh.index as index from tqdm import tqdm from whoosh.fields import Schema, TEXT, ID from whoosh.qparser import QueryParser, OrGroup from semantic_sim import SimServer tokenize_it = Italian().Defaults.create_tokenizer() tokenize_en = English().Defaults.create_tokenizer() wnl = WordNetLemmatizer() punct = string.punctuation.replace('.', '').replace(',', '') def to_shingles(doc, k=5): shingles = set() doc_string = doc.lower() if len(doc_string) <= k: doc_string = doc + 'no_txt_' + str(xxhash.xxh64(str(random.random())).hexdigest()) for i in range(len(doc_string) - k + 1): h = doc_string[i:i+k] shingles.add(h.encode('utf8')) return list(shingles)
import sys import string import spacy from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer from sklearn.base import TransformerMixin from sklearn.pipeline import Pipeline from spacy.lang.it.stop_words import STOP_WORDS from spacy.lang.it import Italian from sklearn import svm from sklearn import neural_network from sklearn import metrics punctuations = string.punctuation nlp = spacy.load("it_core_news_sm") stop_words = spacy.lang.it.stop_words.STOP_WORDS parser = Italian() # Custom transformer using spaCy class predictors(TransformerMixin): def transform(self, X, **transform_params): # Cleaning Text return [clean_text(text) for text in X] def fit(self, X, y=None, **fit_params): return self def get_params(self, deep=True): return {}
def test_issue2179(): """Test that spurious 'extra_labels' aren't created when initializing NER.""" nlp = Italian() ner = nlp.add_pipe("ner") ner.add_label("CITIZENSHIP") nlp.initialize() nlp2 = Italian() nlp2.add_pipe("ner") assert len(nlp2.get_pipe("ner").labels) == 0 model = nlp2.get_pipe("ner").model model.attrs["resize_output"](model, nlp.get_pipe("ner").moves.n_moves) nlp2.from_bytes(nlp.to_bytes()) assert "extra_labels" not in nlp2.get_pipe("ner").cfg assert nlp2.get_pipe("ner").labels == ("CITIZENSHIP", )
elif lang == "nl": return set( get_stop_words("nl") + stopwords.words('dutch') + STOP_LIST_NL) except: print("warning: no stopwords were downloaded. check nltk corpora") print(format_exc()) return set() # load resources _stop_words = load_stoplist() print("Loading spacy model...") _spacy = English() _spacy_fr = French() _spacy_nl = Dutch() _spacy_it = Italian() def get_stoplist(): return _stop_words def lemmatize(text, lowercase=True, lang="en"): """ Return lemmatized text """ if lang == "en": tokens = _spacy(text) elif lang == "fr": tokens = _spacy_fr(text) elif lang == "nl": tokens = _spacy_nl(text)
from spacy.lang.ru import Russian from spacy.lang.zh import Chinese from spacy.lang.ja import Japanese from spacy.lang.ca import Catalan from spacy.lang.eu import Basque from DataHandler import load_df_twitter_sent, load_df_lorelei from util import clean_str as test_clean_str from nltk.corpus import stopwords from util import identity_fn, lang2id language_dict = { 'english': English(), 'spanish': Spanish(), 'french': French(), 'italian': Italian(), 'german': German(), 'russian': Russian(), 'chinese': Chinese(), 'japanese': Japanese(), 'catalan': Catalan(), 'basque': Basque(), } class Tokenizer: def __init__(self, language, tokenizer_method='spacy', remove_stopwords=True, lowercase=True,
def test_issue2179(): """Test that spurious 'extra_labels' aren't created when initializing NER.""" nlp = Italian() ner = nlp.create_pipe("ner") ner.add_label("CITIZENSHIP") nlp.add_pipe(ner) nlp.begin_training() nlp2 = Italian() nlp2.add_pipe(nlp2.create_pipe("ner")) nlp2.from_bytes(nlp.to_bytes()) assert "extra_labels" not in nlp2.get_pipe("ner").cfg assert nlp2.get_pipe("ner").labels == ("CITIZENSHIP",)
def test_issue2482(): """Test we can serialize and deserialize a blank NER or parser model.""" nlp = Italian() nlp.add_pipe(nlp.create_pipe("ner")) b = nlp.to_bytes() Italian().from_bytes(b)