def write_core_format_into_conllup_file(sentences, filepath): print( "Converting {} sentences into CONLLUP format. This requires a text preprocessor for Romanian. If the following function fails please install NLP-Cube (pip3 install nlpcube)." ) from cube.api import Cube cube = Cube(verbose=True) cube.load("ro", tokenization=True, compound_word_expanding=False, tagging=True, lemmatization=True, parsing=True) cube_no_tok = Cube(verbose=True) cube_no_tok.load("ro", tokenization=False, compound_word_expanding=False, tagging=True, lemmatization=True, parsing=True) conllupdataset = [] for sentence in sentences: sentence = process_split_exceptions(sentence) conllupsentence = _conllup_to_core_sentence(sentence, cube, cube_no_tok) conllupdataset.append(conllupsentence) write_file(filepath, conllupdataset)
def prepare_dialogs_sorted_by_lang(dialog_ids, dialog_path, prepared_path, start_date, end_date, additional_options=""): dialog_ids_sorted_by_lang = {"ua": [], "ru": [], "en": []} if dialog_ids[0] == -1: for filename in os.listdir(dialog_path): data = pd.read_csv(f"{dialog_path}/{filename}") lang = detect_data_language(data) dialog_ids_sorted_by_lang[lang].append(filename[:-4]) else: for dialog in dialog_ids: data = pd.read_csv(f"{dialog_path}/{dialog}.csv") lang = detect_data_language(data) dialog_ids_sorted_by_lang[lang].append(dialog) print("dialog_ids_sorted_by_lang") pprint(dialog_ids_sorted_by_lang) n_all_dialogs = sum([ len(dialog_ids_sorted_by_lang[lang]) for lang in dialog_ids_sorted_by_lang.keys() ]) n_dialog = 0 for lang in dialog_ids_sorted_by_lang.keys(): if not dialog_ids_sorted_by_lang[lang]: continue cube = "" if lang == "ua": cube = Cube(verbose=True) cube.load("uk") elif lang == "en": cube = Cube(verbose=True) cube.load("en") for dialog_id in dialog_ids_sorted_by_lang[lang]: if f"{dialog_id}.csv" in os.listdir(prepared_path): print( f"=========WARNING: {dialog_id}.csv already in {prepared_path}" ) n_dialog += 1 continue n_dialog += 1 print( f"\n=======Language {lang}, dialog_id {dialog_id}-- {n_dialog} from {n_all_dialogs}=======" ) prepare_dialogs(lang, cube, dialog_id, prepared_path, dialog_path, start_date, end_date, "words_frequency", additional_options)
def download_model(self): if self.lib.lower() == "stanford": print( "-----------You are going to use Stanford library-----------") if self.lang.lower() == "basque": print( "-------------You are going to use Basque model-------------" ) # MODELS_DIR = '/home/kepa/eu' MODELS_DIR = 'J:\TextSimilarity\eu' stanfordnlp.download('eu', MODELS_DIR) # Download the Basque models elif self.lang.lower() == "english": print( "-------------You are going to use English model-------------" ) MODELS_DIR = '/home/kepa/en' print( "-------------Downloading Stanford Basque model-------------" ) stanfordnlp.download('en', MODELS_DIR) # Download the Basque models elif self.lang.lower() == "spanish": print( "-------------You are going to use Spanish model-------------" ) MODELS_DIR = '/home/kepa/es' stanfordnlp.download('es', MODELS_DIR) # Download the English models else: print("........You cannot use this language...........") elif self.lib.lower() == "cube": print("-----------You are going to use Cube Library-----------") if self.lang.lower() == "basque": cube = Cube(verbose=True) cube.load("eu", "latest") elif self.lang.lower() == "english": cube = Cube(verbose=True) cube.load("en", "latest") elif self.lang.lower() == "spanish": cube = Cube(verbose=True) cube.load("es", "latest") else: print("........You cannot use this language...........") else: print( "You cannot use this library. Introduce a valid library (Cube or Stanford)" )
def main(filename): cube = Cube(verbose=True) cube.load('en') with open('words.txt') as f: word_list = [line.rstrip() for line in f.readlines()] word_set = set(word_list) with open('my_words.txt') as f: my_word_list = [line.rstrip() for line in f.readlines()] my_word_set = set(my_word_list) text = srt_to_text(filename) sentences = cube(text) new_words = [] for sentence in sentences: for entry in sentence: if entry.lemma in word_set and entry.lemma not in my_word_list: if entry.lemma not in new_words: new_words.append(entry.lemma) print('-' * 100) print(f'{len(new_words)} new words are found.') print('-' * 100) for i, word in enumerate(new_words): print(i, word)
def test_4_3_run_model_with_default_external_embeddings(self): print("\n\33[33m{}\33[0m".format("4.3. Run a local model with default external embeddings ...")) from cube.api import Cube cube = Cube(verbose=True) cube.load('my_model', tokenization=True, compound_word_expanding=False, tagging=True, lemmatization=True, parsing=True) text = "I'm a success today because I had a friend who believed in me and I didn't have the heart to let him down. This is a quote by Abraham Lincoln." sentences = cube(text) self.assertTrue(len(sentences)>0) self.assertTrue(len(sentences[0])>0)
def test_2_run_a_local_model(self): print("\n\33[33m{}\33[0m".format("2. Run a local model that does not have embeddings or metadata (running with dummy.vec embeddings) ...")) embeddings = os.path.join(self.root_path, "examples","wiki.dummy.vec") from cube.api import Cube cube = Cube(verbose=True) cube.load('my_model', tokenization=True, compound_word_expanding=False, tagging=True, lemmatization=True, parsing=True, local_models_repository=self.local_model_repo, local_embeddings_file=embeddings) text = "I'm a success today because I had a friend who believed in me and I didn't have the heart to let him down. This is a quote by Abraham Lincoln." sentences = cube(text) self.assertTrue(len(sentences)>0) self.assertTrue(len(sentences[0])>0)
def test_1_2_download_and_run_an_online_model_specific_version(self): print("\n\33[33m{}\33[0m".format("1.2. Loading an online model (sme, 1.0) ...")) from cube.api import Cube cube = Cube(verbose=True) cube.load('sme', version='1.0', tokenization=True, compound_word_expanding=False, tagging=False, lemmatization=False, parsing=False) cube.metadata.info() text = "I'm a success today because I had a friend who believed in me and I didn't have the heart to let him down. This is a quote by Abraham Lincoln." sentences = cube(text) self.assertTrue(len(sentences)>0) self.assertTrue(len(sentences[0])>0)
def test_3_3_run_model_with_manual_embeddings(self): print("\n\33[33m{}\33[0m".format("3.3. Run a local model with manual embeddings ...")) embeddings = os.path.join(self.root_path, "examples","wiki.dummy.vec") print("\t\tPath to local manual embeddings file: "+embeddings) from cube.api import Cube cube = Cube(verbose=True) cube.load('my_model', tokenization=True, compound_word_expanding=False, tagging=True, lemmatization=True, parsing=True, local_embeddings_file=embeddings) text = "I'm a success today because I had a friend who believed in me and I didn't have the heart to let him down. This is a quote by Abraham Lincoln." sentences = cube(text) self.assertTrue(len(sentences)>0) self.assertTrue(len(sentences[0])>0)
def __init__(self, iso, use_gpu=False): from pos_tagger.engines.cubenlp_pos.CubeNLPPOS import \ DSupportedISOs assert iso in DSupportedISOs # Note the GPU argument - it may be worth putting # this on the PC with a more powerful one from cube.api import Cube cube_inst = self.cube_inst = Cube(verbose=True, use_gpu=use_gpu) # Chinese doesn't seem to work well with version 1.0, # [[??? <- I think this means 1.1??]] # despite the scores not showing much difference cube_inst.load(DSupportedISOs[iso], version='1.0' if iso == 'zh' else '1.1') EngineInstance.__init__(self, iso, use_gpu)
def create_conll_sentences(file_path): print("*"*25 + " Working on transforming the input file '{}' to CoNLL format ".format(file_path) + "*"*25 + "\n") print("Reading the input file...") with open(file_path, "r", encoding="utf-8") as file: text = file.read() print("Loading the 'ro' nlp-cube model...") cube = Cube(verbose=False) cube.load("ro") print("Creating the CoNLL sentences...") sentences = cube(text) print("\n" + "*"*124 + "\n") return sentences
def get_lemmatized_vocabulary(unlemmatized_voc, epi, lang): if os.path.isfile('lemmatized_' + lang + '.p'): voc = p.load(open('lemmatized_' + lang + '.p', 'rb')) return voc # Lemmatizer lang_acron = { 'gothic': 'got', 'latin': 'la', 'italian': 'it', 'german': 'de', 'greek': 'grc', 'english': 'eng' } cube = Cube(verbose=False) cube.load(lang_acron[lang]) voc = {} keys = list(unlemmatized_voc.keys()) for i in tqdm(range(len(keys))): w = keys[i] sents = cube(w) if type(sents[0]) == list: for sent in sents: for token in sent: if token.lemma != '_': voc[w] = [ token.lemma, epi.transliterate(token.lemma), 'L' ] else: voc[w] = [ token.word, epi.transliterate(token.word), 'T' ] else: for token in sents: if token.lemma != '_': voc[w] = [token.lemma, epi.transliterate(token.lemma), 'L'] else: voc[w] = [token.word, epi.transliterate(token.word), 'T'] p.dump(voc, open('lemmatized_' + lang + '.p', 'wb')) return voc
def download_model(self): if self.lib.lower() == "stanford": print("-----------You are going to use Stanford library-----------") if self.lang.lower() == "basque": print("-------------You are going to use Basque model-------------") # MODELS_DIR = '/home/edercarbajo/eu' MODELS_DIR = 'J:\TextSimilarity\eu' stanfordnlp.download('eu', MODELS_DIR) # Download the Basque models # config = {'processors': 'tokenize,pos,lemma,depparse', # Comma-separated list of processors to use # 'lang': 'eu', # Language code for the language to build the Pipeline in # 'tokenize_model_path': '/home/edercarbajo/eu/eu_bdt_models/eu_bdt_tokenizer.pt', # # Processor-specific arguments are set with keys "{processor_name}_{argument_name}" # 'pos_model_path': '/home/edercarbajo/eu/eu_bdt_models/eu_bdt_tagger.pt', # 'pos_pretrain_path': '/home/edercarbajo/eu/eu_bdt_models/eu_bdt.pretrain.pt', # 'lemma_model_path': '/home/edercarbajo/eu/eu_bdt_models/eu_bdt_lemmatizer.pt', # 'depparse_model_path': '/home/edercarbajo/eu/eu_bdt_models/eu_bdt_parser.pt', # 'depparse_pretrain_path': '/home/edercarbajo/eu/eu_bdt_models/eu_bdt.pretrain.pt' # } config = {'processors': 'tokenize,pos,lemma,depparse', # Comma-separated list of processors to use 'lang': 'eu', # Language code for the language to build the Pipeline in 'tokenize_model_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt_tokenizer.pt', # Processor-specific arguments are set with keys "{processor_name}_{argument_name}" 'pos_model_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt_tagger.pt', 'pos_pretrain_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt.pretrain.pt', 'lemma_model_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt_lemmatizer.pt', 'depparse_model_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt_parser.pt', 'depparse_pretrain_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt.pretrain.pt' } self.parser = stanfordnlp.Pipeline(**config) else: print("............Working...........") elif self.lib.lower() == "cube": print("-----------You are going to use Cube Library-----------") if self.lang.lower() == "basque": cube = Cube(verbose=True) cube.load("eu", "latest") self.parser = cube else: print("............Working...........") else: print("You cannot use this library. Introduce a valid library (Cube or Stanford)")
def start(): from analyzer import Analyzer p = ArgumentParser(description="python3 ./main.py -f \"laginak/*.doc.txt\" ") optional = p._action_groups.pop() # Edited this line required = p.add_argument_group('Required arguments') required.add_argument("-f", "--files", nargs='+', help="Files to analyze (in .txt, .odt, .doc or .docx format)") optional.add_argument('-a', '--all', action='store_true', help="Generate a CSV file with all the results") optional.add_argument('-s', '--similarity', action='store_true', help="Calculate similarity (max. 5 files)") p._action_groups.append(optional) opts = p.parse_args() FileLoader.load_files(opts.files) FileLoader.load_irregular_verbs_list() FileLoader.load_dale_chall_list() FileLoader.load_connectives_list() FileLoader.load_oxford_word_list() cube = Cube(verbose=True) # Cargar modelo Cube cube.load("en", "latest") df_row = None ### Files will be created in this folder path = Printer.create_directory(FileLoader.files[0]) file_num = 0 total = len(FileLoader.files) for input in FileLoader.files: texto = Analyzer.process_text(input=input) # Analizar a = Analyzer(texto, input, cube) i = a.analyze(opts.similarity) df = a.create_dataframe() prediction = a.predict_dificulty(df) file_num += 1 p = Printer(input, i) p.print_info(opts.similarity, prediction, file_num, total) if opts.all: df_row = p.write_in_full_csv(df_row, opts.similarity) p.generate_csv(path, prediction, opts.similarity) if opts.all: df_row.to_csv(os.path.join(path, "full_results_aztertest.csv"), encoding='utf-8', index=False)
def createApp(self): self._cubeInst = Cube(verbose=True)
def cli(ctx, language): cube = Cube(verbose=True) cube.load(language) ctx.ensure_object(dict) ctx.obj['CUBE'] = cube
def map_bibles(f1, f2s, voc, l1='gothic', cube1=True, cube2=False, lemmatizer={}, expand_voc=False): f1_dict = load_bible(open(f1, 'r')) lemma_l1 = False if cube1 == False else True count = 0 mapped = {} all_words_count = 0 found_words = {} unfound_words = {} for book in f1_dict: if book not in mapped: mapped[book] = {} for chapter in f1_dict[book]: if chapter not in mapped[book]: mapped[book][chapter] = {} for verse in f1_dict[book][chapter]: if verse not in mapped[book][chapter]: mapped[book][chapter][verse] = {} count += 1 if count % 500 == 0: print(count) mapped[book][chapter][verse][l1] = f1_dict[book][chapter][ verse] if lemma_l1: mapped[book][chapter][verse][l1 + '_analyzed'] = analyze( f1_dict[book][chapter][verse], cube1, lang='got') else: mapped[book][chapter][verse][ l1 + '_analyzed'] = fake_analyze( f1_dict[book][chapter][verse], lang='got', lemmatizer=lemmatizer) if l1 == 'got': mapped[book][chapter][verse][l1 + '_translation'] = {} lemmas = list( list( zip(*mapped[book][chapter][verse][ l1 + '_analyzed']))[2]) for lang in all_languages: if lang not in found_words: found_words[lang] = {} if lang not in unfound_words: unfound_words[lang] = {} if lang == 'Got': mapped[book][chapter][verse][ l1 + '_translation'][lang] = lemmas mapped[book][chapter][verse][l1 + '_translation'][ lang + '_script'] = [ gothic_script_transformer(t) for t in lemmas ] mapped[book][chapter][verse][l1 + '_translation'][ lang + '_ipa'] = [ ipa_transformer(t, 'gothic') for t in lemmas ] else: if lang not in mapped[book][chapter][verse][ l1 + '_translation']: mapped[book][chapter][verse][ l1 + '_translation'][lang] = [] for word in lemmas: all_words_count += 1 if word in voc and lang in voc[word]: #if lang == 'Lat': # print(word, voc[word][lang]) if word not in found_words[lang]: found_words[lang][word] = 0 found_words[lang][word] += 1 mapped[book][chapter][verse][ l1 + '_translation'][lang].append( voc[word][lang]) else: if word not in unfound_words[lang]: unfound_words[lang][word] = 0 unfound_words[lang][word] += 1 mapped[book][chapter][verse][ l1 + '_translation'][lang].append([]) for lang in found_words: found = sum([found_words[lang][w] for w in found_words[lang]]) unfound = sum([unfound_words[lang][w] for w in unfound_words[lang]]) print('token', lang, found, unfound, '%.4f' % (float(found) / (found + unfound + 1))) found = len(found_words[lang]) unfound = len(unfound_words[lang]) print('\ttype', lang, found, unfound, '%.4f' % (float(found) / (found + unfound + 1))) print('All words: {}'.format(all_words_count)) for f in f2s: l2 = f.split('/')[0] langs_epitran = { 'german': 'deu-Latn', 'italian': 'ita-Latn', 'latin': 'ita-Latn', 'spanish': 'spa-Latn', 'english': 'eng-Latn' } if l2 in langs_epitran: epi = epitran.Epitran(langs_epitran[l2]) if cube2: lemma_l2 = lemma_l1 cube2 = Cube(verbose=False) cube2.load(lang_acron[l2]) #if f == 'greek/greek_byzantine_2000_utf8.txt': # pdb.set_trace() f2_dict = load_bible(open(f, 'r')) lemma_l2 = False if cube1 == False else True for book in mapped: for chapter in mapped[book]: for verse in mapped[book][chapter]: if book not in f2_dict or chapter not in f2_dict[ book] or verse not in f2_dict[book][chapter]: #pdb.set_trace() continue if f2_dict[book][chapter][verse] in ['', '[]', []]: continue #print(f2_dict[book][chapter][verse]) #mapped_words = {'german':'deu-Latn', 'italian':'ita-Latn', 'latin':'ita-Latn', 'spanish':'Es', 'greek':'Gre'} if lemma_l2: mapped[book][chapter][verse][ l2 + '_analyzed'] = analyze( f2_dict[book][chapter][verse], cube2, lang=l2) else: mapped[book][chapter][verse][ l2 + '_analyzed'] = fake_analyze( f2_dict[book][chapter][verse], lang=l2) # IPA #try: lemmas = [ x for x in (list( zip(*mapped[book][chapter][verse][l2 + '_analyzed']))[2] ) ] #except: #pdb.set_trace() mapped[book][chapter][verse][l2] = lemmas if l2 == 'greek': mapped[book][chapter][verse][l2 + '_ipa'] = [ ipa_transformer(l, 'greek') for l in lemmas ] #.split() elif l2 == 'old_english': try: mapped[book][chapter][verse][l2 + '_ipa'] = [ oe(l) for l in lemmas ] #.split() except: pass elif l2 == 'english': try: mapped[book][chapter][verse][l2 + '_ipa'] = [ oe(l) for l in lemmas ] #.split() #gen_mods.get_final(gen_mods.getIPA_CMU(f2_dict[book][chapter][verse])) except: pass else: mapped[book][chapter][verse][l2 + '_ipa'] = [ epi.transliterate(l) for l in lemmas ] #.split() expand_voc = False if l2 != 'Got' and expand_voc: expand_voc_by_distance(voc, mapped, book, chapter, verse, l2) #pdb.set_trace() return mapped
parser.add_option('--port', action='store', dest='port', type='int', default=8080, help='Binding port for web service (default: 8080)') parser.add_option('--host', action='store', dest='host', default='0.0.0.0', help='Binding IP for server (default: 0.0.0.0)') parser.add_option( '--lang', action='append', dest='languages', default=[], help= 'Preload language. You can use this param multiple times: --lang en --lang fr ... (default is just ["en"])' ) (params, _) = parser.parse_args(sys.argv) if len(params.languages) == 0: params.languages = ['en'] for lang in params.languages: lang2cube[lang] = Cube(verbose=True) lang2cube[lang].load(lang) app.run(port=params.port, host=params.host)
's', 's', 's', 's', 's', 's', 'm', 'm', 's', 's', 'm', 's', 'm', 's', 'm', 's', 's', 'm', 'm', 'm', 's', 's', 's', 's', 's', 'm', 's', 's', 's', 's', 's', 's', 's', 'm', 's', 's', 's', 's', 's', 's', 'm', 's', 'm', 's', 'm', 's', 'x', 's', 's', 's', 's', 's', 's', 'm', 's', 's', 's', 's', 'm', 's', 's', 's', 'm', 'm', 's', 's', 's', 's', 's', 's', 's', 's', 'm', 's', 'm', 'm', 's', 's', 'm', 's', 's', 's', 'm', 'm', 's', 's', 'm', 's', 's', 'm', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 's', 'm', 's', 's', 's', 's', 's', 's', 's', 'm', 's', 's', 'm', 's', 's', 's', 's', 's', 's', 'm', 's', 'm', 's', 's', 's', 's', 'm', 's', 's', 's', 'm', 'm', 's', 's', 's', 'm' ] from cube.api import Cube cube = Cube(verbose=True) cube.load("ro", tokenization=True, compound_word_expanding=False, tagging=True, lemmatization=True, parsing=True) cube_no_tok = Cube(verbose=True) cube_no_tok.load("ro", tokenization=False, compound_word_expanding=False, tagging=True, lemmatization=True, parsing=True) multi_sentence_count = 0 errors = 0
for LSentence in LSentences: for entry in LSentence: print(entry) print("") if __name__ == '__main__': if True: found_nl = False for iso in CubeNLPPOS.get_L_supported_isos(None): if iso not in ('nno', 'nnb'): continue from cube.api import Cube # import the Cube object cube = Cube(verbose=True) # initialize it cube.load(DSupportedISOs[iso]) print_pos( 'id', 'Tahap pertama konflik ini dapat disebut "Perang Kemerdekaan Belanda".' ) print_pos( 'en', 'The first phase of the conflict can be considered the Dutch War of Independence.' ) print_pos('id', 'Saya tidak dapat memakan ini.') print_pos('en', 'I can\'t eat this.') print_pos('zh', '猴子高兴,实验人员也高兴。')
import tnkeeh as tn from farasa.segmenter import FarasaSegmenter # for bulgarian and turkish from cube.api import Cube en_tok = MosesTokenizer(lang="en") en_normalizer = MosesPunctNormalizer() # TODO: change hardcoding of jar file to a arg from cli rdrsegmenter = VnCoreNLP("vncorenlp/VnCoreNLP-1.1.1.jar", annotators="wseg", max_heap_size="-Xmx500m") ar_segmenter = FarasaSegmenter() bg_cube = Cube(verbose=False) bg_cube.load("bg") tr_cube = Cube(verbose=False) tr_cube.load("tr") def clean_ar_text( text, segment=False, remove_special_chars=False, remove_english=False, normalize=False, remove_diacritics=False, excluded_chars=[], remove_tatweel=False,
# implement NLP-Cube https://github.com/adobe/NLP-Cube for sent and word tokenization, and lemma from cube.api import Cube # import the Cube object cube = Cube(verbose=False) # initialize it cube.load("en") from rusenttokenize import ru_sent_tokenize import re class mytokenizer: def bracket_mask(self, text): text_temp = re.sub("\[", "<<", text) text_temp = re.sub("\]", ">>", text_temp) text_temp = re.sub("\+", "===", text_temp) parenthesis = re.findall("\([^()]*\)", text_temp) bracket = re.findall("\[[^\[\]]*\]", text) if not parenthesis and not bracket: return text for p in parenthesis: p = re.sub("[\(\)]", "", p) parts = [re.sub("\.\s*$", "", s) for s in ru_sent_tokenize(p)] new_p = " ; ".join(parts) #print ("P:", p) #print("new_p:",new_p,"\n") #print("text_temp:",text_temp) text_temp = re.sub(p, new_p, text_temp) text = re.sub("<<", "[", text_temp) text = re.sub(">>", "]", text)
# filter import os import csv import json from filter.filter import filter_spam # lemmatizer from lemmatizer.lemmatizer import lemmatize from cube.api import Cube # detection sys.path.insert(0, './detector/') from detect_events import main as detect_events lemmatizer = Cube(verbose=True) lemmatizer.load("es", tokenization=False, parsing=False) class Streamer(luigi.ExternalTask): time_slice = luigi.parameter.DateMinuteParameter(interval=30) def output(self): fname = '../data/streaming/{}.csv'.format(self.time_slice) # print('Requires: {}'.format(fname)) return luigi.LocalTarget(fname) class Preprocess(luigi.Task): time_slice = luigi.parameter.DateMinuteParameter( interval=30, default=datetime.datetime.today())
from cube.api import Cube cube = Cube(verbose=True) cube.load( 'ro', tokenization=True, compound_word_expanding=False, tagging=True, lemmatization=True, parsing=True )
def load_model(self): if self.lib.lower() == "stanford": print( "-----------You are going to use Stanford library-----------") if self.lang.lower() == "basque": print( "-------------You are going to use Basque model-------------" ) # config = {'processors': 'tokenize,pos,lemma,depparse', # Comma-separated list of processors to use # 'lang': 'eu', # Language code for the language to build the Pipeline in # 'tokenize_model_path': '/home/kepa/eu/eu_bdt_models/eu_bdt_tokenizer.pt', # # Processor-specific arguments are set with keys "{processor_name}_{argument_name}" # 'pos_model_path': '/home/kepa/eu/eu_bdt_models/eu_bdt_tagger.pt', # 'pos_pretrain_path': '/home/kepa/eu/eu_bdt_models/eu_bdt.pretrain.pt', # 'lemma_model_path': '/home/kepa/eu/eu_bdt_models/eu_bdt_lemmatizer.pt', # 'depparse_model_path': '/home/kepa/eu/eu_bdt_models/eu_bdt_parser.pt', # 'depparse_pretrain_path': '/home/kepa/eu/eu_bdt_models/eu_bdt.pretrain.pt' # } config = { 'processors': 'tokenize,pos,lemma,depparse', # Comma-separated list of processors to use 'lang': 'eu', # Language code for the language to build the Pipeline in 'tokenize_model_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt_tokenizer.pt', # Processor-specific arguments are set with keys "{processor_name}_{argument_name}" 'pos_model_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt_tagger.pt', 'pos_pretrain_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt.pretrain.pt', 'lemma_model_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt_lemmatizer.pt', 'depparse_model_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt_parser.pt', 'depparse_pretrain_path': 'J:\TextSimilarity\eu\eu_bdt_models\eu_bdt.pretrain.pt' } self.parser = stanfordnlp.Pipeline(**config) elif self.lang.lower() == "english": print( "-------------You are going to use English model-------------" ) config = { 'processors': 'tokenize,mwt,pos,lemma,depparse', # Comma-separated list of processors to use 'lang': 'en', # Language code for the language to build the Pipeline in 'tokenize_model_path': '/home/kepa/en/en_ewt_models/en_ewt_tokenizer.pt', # Processor-specific arguments are set with keys "{processor_name}_{argument_name}" #'mwt_model_path': './fr_gsd_models/fr_gsd_mwt_expander.pt', 'pos_model_path': '/home/kepa/en/en_ewt_models/en_ewt_tagger.pt', 'pos_pretrain_path': '/home/kepa/en/en_ewt_models/en_ewt.pretrain.pt', 'lemma_model_path': '/home/kepa/en/en_ewt_models/en_ewt_lemmatizer.pt', 'depparse_model_path': '/home/kepa/en/en_ewt_models/en_ewt_parser.pt', 'depparse_pretrain_path': '/home/kepa/en/en_ewt_models/en_ewt.pretrain.pt' } self.parser = stanfordnlp.Pipeline(**config) elif self.lang.lower() == "spanish": print( "-------------You are going to use Spanish model-------------" ) config = { 'processors': 'tokenize,pos,lemma,depparse', # Comma-separated list of processors to use 'lang': 'es', # Language code for the language to build the Pipeline in 'tokenize_model_path': '/home/kepa/es/es_ancora_models/es_ancora_tokenizer.pt', # Processor-specific arguments are set with keys "{processor_name}_{argument_name}" 'pos_model_path': '/home/kepa/es/es_ancora_models/es_ancora_tagger.pt', 'pos_pretrain_path': '/home/kepa/es/es_ancora_models/es_ancora.pretrain.pt', 'lemma_model_path': '/home/kepa/es/es_ancora_models/es_ancora_lemmatizer.pt', 'depparse_model_path': '/home/kepa/es/es_ancora_models/es_ancora_parser.pt', 'depparse_pretrain_path': '/home/kepa/es/es_ancora_models/es_ancora.pretrain.pt' } self.parser = stanfordnlp.Pipeline(**config) else: print("........You cannot use this language...........") elif self.lib.lower() == "cube": print("-----------You are going to use Cube Library-----------") if self.lang.lower() == "basque": #load(self, language_code, version="latest",local_models_repository=None, #local_embeddings_file=None, tokenization=True, compound_word_expanding=False, #tagging=True, lemmatization=True, parsing=True). #Ejemplo:load("es",tokenization=False, parsing=False) cube = Cube(verbose=True) cube.load("eu", "latest") elif self.lang.lower() == "english": cube = Cube(verbose=True) cube.load("en", "latest") elif self.lang.lower() == "spanish": cube = Cube(verbose=True) cube.load("es", "latest") else: print("........You cannot use this language...........") else: print( "You cannot use this library. Introduce a valid library (Cube or Stanford)" )
def create_pickle(): cube = Cube(verbose=True) cube.load("ja") with open('cube.pickle', mode='wb') as wh: pickle.dump(cube, wh)