def parse_test(params):
if params.test == "parser":
print ("Running " + params.test)
print ("==PARAMETERS==")
print ("EMBEDDINGS: " + params.embeddings)
print ("MODEL FILE: " + params.model_base)
print ("DECODER: " + params.decoder)
print ("OUTPUT: " + params.output_file)
print ("CONFIG FILE: " + str(params.config))
print ("==============\n")
testset = Dataset(params.test_file)
encodings = Encodings()
encodings.load(params.model_base + ".encodings")
encodings.update_wordlist(testset)
print ("Updated word list: " + str(len(encodings.word_list)))
config = ParserConfig(filename=params.config)
embeddings = WordEmbeddings()
embeddings.read_from_file(params.embeddings, encodings.word_list)
parser = BDRNNParser(config, encodings, embeddings)
parser.load(params.model_base + ".bestUAS")
if params.decoder == 'mst':
print ("!!!!!!!!!!!!!!!!!!!!!!!!!USING MST DECODER")
from graph.decoders import MSTDecoder
parser.decoder = MSTDecoder()
f = fopen(params.output_file, "w")
last_proc = 0
index = 0
for seq in testset.sequences:
index += 1
proc = index * 100 / len(testset.sequences)
if proc % 5 == 0 and proc != last_proc:
last_proc = proc
sys.stdout.write(" " + str(proc))
sys.stdout.flush()
rez = parser.tag(seq)
iSeq = 0
iRez = 0
while iSeq < len(seq):
while seq[iSeq].is_compound_entry:
iSeq += 1
seq[iSeq].xpos = rez[iRez].xpos
seq[iSeq].upos = rez[iRez].upos
seq[iSeq].attrs = rez[iRez].attrs
seq[iSeq].head = rez[iRez].head
seq[iSeq].label = rez[iRez].label
seq[iSeq].lemma = rez[iRez].lemma
iSeq += 1
iRez += 1
for entry in seq:
f.write(str(entry.index) + "\t" + str(entry.word) + "\t" + str(entry.lemma) + "\t" + str(
entry.upos) + "\t" + str(entry.xpos) + "\t" + str(entry.attrs) + "\t" + str(
entry.head) + "\t" + str(entry.label) + "\t" + str(entry.deps) + "\t" + str(
entry.space_after) + "\n")
f.write("\n")
f.close()
sys.stdout.write("\n")
class EmbeddedWebserver:
def __init__(self, embeddings, port=80, tokenization=None, lemma=None, tagging=None, parsing=None):
global singletonServer
singletonServer = self
if tokenization is not None:
sys.stdout.write("Loading tokenization model from " + tokenization)
sys.stdout.flush()
from generic_networks.tokenizers import TieredTokenizer
from io_utils.config import TieredTokenizerConfig
from io_utils.encodings import Encodings
tok_encodings = Encodings()
tok_encodings.load(tokenization + ".encodings")
tok_config = TieredTokenizerConfig()
tok_config.load(tokenization + ".conf")
self.tokenizer = TieredTokenizer(tok_config, tok_encodings, embeddings, runtime=True)
self.tokenizer.load(tokenization)
if parsing is not None:
sys.stdout.write("Loading parsing model from " + parsing)
from generic_networks.parsers import BDRNNParser
from io_utils.config import ParserConfig
from io_utils.encodings import Encodings
parse_encodings = Encodings()
parse_encodings.load(parsing + ".encodings")
parse_config = ParserConfig()
parse_config.load(parsing + ".conf")
self.parser = BDRNNParser(parse_config, parse_encodings, embeddings, runtime=True)
self.parser.load(parsing + ".bestUAS")
if lemma is not None:
sys.stdout.write("Loading lemma model from " + lemma)
from generic_networks.lemmatizers import BDRNNLemmatizer
from io_utils.config import LemmatizerConfig
from io_utils.encodings import Encodings
lemma_encodings = Encodings()
lemma_encodings.load(lemma + ".encodings")
lemma_config = LemmatizerConfig()
lemma_config.load(lemma + ".conf")
self.lemmatizer = BDRNNLemmatizer(lemma_config, lemma_encodings, embeddings, runtime=True)
self.lemmatizer.load(lemma + ".bestACC")
else:
self.lemmatizer = None
global app
app.run(port=port)
self.port = port
def parse_run(params):
sys.stdout.write("\nINPUT FILE: " + params.input_file)
sys.stdout.write("\nOUTPUT FILE: " + params.output_file)
sys.stdout.write("\nMODELS FILE: " + params.models + "\n")
sys.stdout.flush()
components = params.run.split(",")
tokenize = True if "tokenizer" in components else False
compound = True if "compound" in components else False
lemmatize = True if "lemmatizer" in components else False
tag = True if "tagger" in components else False
parse = True if "parser" in components else False
# common elements load
sys.stdout.write("\nLoading embeddings : " + params.embeddings + " ...\n")
embeddings = WordEmbeddings()
embeddings.read_from_file(params.embeddings, None)
encodings = None
if tokenize == True:
if not os.path.isfile(
os.path.join(params.models, "tokenizer-tok.bestAcc")):
sys.stdout.write(
"\n\tTokenizer model not found! (" +
os.path.join(params.models, "tokenizer-tok.bestAcc") + ")")
sys.stdout.flush()
sys.exit(1)
sys.stdout.write("\n\tTokenization enabled.\n")
tokenizer_encodings = Encodings(verbose=False)
tokenizer_encodings.load(
os.path.join(params.models, "tokenizer.encodings"))
if compound == True:
if not os.path.isfile(os.path.join(params.models, "compound.bestAcc")):
sys.stdout.write("\n\tCompound word expander model not found!")
sys.stdout.flush()
sys.exit(1)
sys.stdout.write("\n\tCompound word expander enabled.\n")
if encodings == None:
encodings = Encodings(verbose=False)
encodings.load(os.path.join(params.models, "lemmatizer.encodings"))
if lemmatize == True:
if not os.path.isfile(os.path.join(params.models,
"lemmatizer.bestACC")):
sys.stdout.write("\n\tLemmatization model not found!")
sys.stdout.flush()
sys.exit(1)
sys.stdout.write("\n\tLemmatization enabled.\n")
if encodings == None:
encodings = Encodings(verbose=False)
encodings.load(os.path.join(params.models, "lemmatizer.encodings"))
if tag == True:
if not os.path.isfile(os.path.join(params.models,
"tagger.bestOVERALL")):
sys.stdout.write("\n\tTagger model not found!")
sys.stdout.flush()
sys.exit(1)
sys.stdout.write("\n\tTagger enabled.\n")
if encodings == None:
encodings = Encodings(verbose=False)
encodings.load(os.path.join(params.models, "tagger.encodings"))
if parse == True:
if not os.path.isfile(os.path.join(params.models, "parser.bestUAS")):
sys.stdout.write("\n\tParser model not found!")
sys.stdout.flush()
sys.exit(1)
sys.stdout.write("\n\tParser enabled.\n")
if encodings == None:
encodings = Encodings(verbose=False)
encodings.load(os.path.join(params.models, "parser.encodings"))
sequences = None
if tokenize:
sys.stdout.write("\nTokenizing " + params.input_file + " ... \n\t")
sys.stdout.flush()
from io_utils.config import TieredTokenizerConfig
from generic_networks.tokenizers import TieredTokenizer
config = TieredTokenizerConfig(
os.path.join(params.models, "tokenizer.conf"))
tokenizer_object = TieredTokenizer(config,
tokenizer_encodings,
embeddings,
runtime=True)
tokenizer_object.load(os.path.join(params.models, "tokenizer"))
with open(params.input_file, 'r') as file:
lines = file.readlines()
# analyze use of spaces in first part of the file
test = ""
useSpaces = " "
cnt = 0
while True:
test = test + lines[cnt]
# print(lines[cnt])
if cnt >= len(lines) or cnt > 5:
break
cnt += 1
if float(test.count(' ')) / float(len(test)) < 0.02:
useSpaces = ""
# print (str(float(test.count(' '))/float(len(test))))
i = -1
input_string = ""
sequences = []
while i < len(lines) - 1:
i += 1
input_string = input_string + lines[i].replace("\r", "").replace(
"\n", "").strip() + useSpaces
if lines[i].strip() == "" or i == len(lines) - 1: # end of block
if input_string.strip() != "":
sequences += tokenizer_object.tokenize(input_string)
input_string = ""
del tokenizer_object # free memory
else:
ds = Dataset(params.input_file)
sequences = ds.sequences
sys.stdout.write(" done\n")
sys.stdout.flush()
if compound:
sys.stdout.write("\nCompound word expanding " + params.input_file +
" ... \n\t")
sys.stdout.flush()
from generic_networks.token_expanders import CompoundWordExpander
from io_utils.config import CompoundWordConfig
config = CompoundWordConfig(
os.path.join(params.models, "compound.conf"))
compoundwordexpander_object = CompoundWordExpander(config,
encodings,
embeddings,
runtime=True)
compoundwordexpander_object.load(
os.path.join(params.models, "compound.bestAcc"))
sequences = compoundwordexpander_object.expand_sequences(sequences)
del compoundwordexpander_object # free memory
sys.stdout.write(" done\n")
sys.stdout.flush()
if parse == True:
sys.stdout.write("\nParsing " + params.input_file + " ... \n\t")
sys.stdout.flush()
from io_utils.config import ParserConfig
from generic_networks.parsers import BDRNNParser
config = ParserConfig(os.path.join(params.models, "parser.conf"))
parser_object = BDRNNParser(config,
encodings,
embeddings,
runtime=True)
parser_object.load(os.path.join(params.models, "parser.bestUAS"))
sequences = parser_object.parse_sequences(sequences)
del parser_object # free memory
sys.stdout.write(" done\n")
sys.stdout.flush()
if tag == True:
sys.stdout.write("\nTagging " + params.input_file + " ... \n\t")
sys.stdout.flush()
from io_utils.config import TaggerConfig
from generic_networks.taggers import BDRNNTagger
config = TaggerConfig(os.path.join(params.models, "tagger.conf"))
tagger_object_UPOS = BDRNNTagger(config,
encodings,
embeddings,
runtime=True)
tagger_object_UPOS.load(os.path.join(params.models, "tagger.bestUPOS"))
tagger_object_XPOS = BDRNNTagger(config,
encodings,
embeddings,
runtime=True)
tagger_object_XPOS.load(os.path.join(params.models, "tagger.bestXPOS"))
tagger_object_ATTRS = BDRNNTagger(config,
encodings,
embeddings,
runtime=True)
tagger_object_ATTRS.load(
os.path.join(params.models, "tagger.bestATTRS"))
new_sequences = []
for sequence in sequences:
new_sequence = copy.deepcopy(sequence)
predicted_tags_UPOS = tagger_object_UPOS.tag(new_sequence)
predicted_tags_XPOS = tagger_object_XPOS.tag(new_sequence)
predicted_tags_ATTRS = tagger_object_ATTRS.tag(new_sequence)
for entryIndex in range(len(sequence)):
new_sequence[entryIndex].upos = predicted_tags_UPOS[
entryIndex][0]
new_sequence[entryIndex].xpos = predicted_tags_XPOS[
entryIndex][1]
new_sequence[entryIndex].attrs = predicted_tags_ATTRS[
entryIndex][2]
new_sequences.append(new_sequence)
sequences = copy.deepcopy(new_sequences)
del tagger_object_UPOS # free memory
del tagger_object_XPOS # free memory
del tagger_object_ATTRS # free memory
sys.stdout.write(" done\n")
sys.stdout.flush()
if lemmatize:
sys.stdout.write("\nLemmatizing " + params.input_file + " ... \n\t")
sys.stdout.flush()
from generic_networks.lemmatizers import FSTLemmatizer
from io_utils.config import LemmatizerConfig
config = LemmatizerConfig(
os.path.join(params.models, "lemmatizer.conf"))
lemmatizer_object = FSTLemmatizer(config,
encodings,
embeddings,
runtime=True)
lemmatizer_object.load(
os.path.join(params.models, "lemmatizer.bestACC"))
sequences = lemmatizer_object.lemmatize_sequences(sequences)
del lemmatizer_object # free memory
sys.stdout.write(" done\n")
sys.stdout.flush()
output_dataset = Dataset()
output_dataset.sequences = sequences
output_dataset.write(params.output_file)
class Cube(object):
def __init__(self, verbose=False):
"""
Create an empty instance for Cube
Before it can be used, you must call @method load with @param language_code set to your target language
"""
self._loaded = False
self._verbose = verbose
self._tokenizer = None # tokenizer object, default is None
self._compound_word_expander = False # compound word expander, default is None
self._lemmatizer = False # lemmatizer object, default is None
self._parser = False # parser object, default is None
self._tagger = False # tagger object, default is None
self.embeddings = None # ?? needed?
self.metadata = ModelMetadata()
self._model_repository = "models"
self._embeddings_repository = os.path.join("models", "embeddings")
#self.model_store = ModelStore() # needed???
def load(self,
language_code,
version="latest",
tokenization=True,
compound_word_expanding=False,
tagging=True,
lemmatization=True,
parsing=True):
"""
Loads the pipeline with all available models for the target language.
@param lang_code: Target language code. See http://opensource.adobe.com/NLP-Cube/ for available languages and their codes
@param version: "latest" to get the latest version, or other specific version in like "1.0", "2.1", etc .
"""
# Initialize a ModelStore object
model_store_object = ModelStore(disk_path=self._model_repository)
# Find a local model or download it if it does not exist, returning the local model folder path
model_folder_path = model_store_object.find(lang_code=language_code,
version=version,
verbose=self._verbose)
# Load metadata from the model
self.metadata.read(os.path.join(model_folder_path, "metadata.json"))
# Load embeddings
embeddings = WordEmbeddings(verbose=False)
if self._verbose:
sys.stdout.write('\tLoading embeddings... \n')
embeddings.read_from_file(os.path.join(
self._embeddings_repository, self.metadata.embeddings_file_name),
None,
full_load=False)
# 1. Load tokenizer
if tokenization:
if not os.path.isfile(
os.path.join(model_folder_path, 'tokenizer-tok.bestAcc')):
sys.stdout.write(
'\tTokenization is not available on this model. \n')
else:
if self._verbose:
sys.stdout.write('\tLoading tokenization model ...\n')
tokenizer_encodings = Encodings(verbose=False)
tokenizer_encodings.load(
os.path.join(model_folder_path, 'tokenizer.encodings'))
config = TieredTokenizerConfig(
os.path.join(model_folder_path, 'tokenizer.conf'))
self._tokenizer = TieredTokenizer(config,
tokenizer_encodings,
embeddings,
runtime=True)
self._tokenizer.load(
os.path.join(model_folder_path, 'tokenizer'))
# 3. Load compound
if compound_word_expanding:
if not os.path.isfile(
os.path.join(model_folder_path, 'compound.bestAcc')):
if self._verbose: # supress warning here because many languages do not have compund words
sys.stdout.write(
'\tCompound word expansion is not available on this model. \n'
)
else:
if self._verbose:
sys.stdout.write(
'\tLoading compound word expander model ...\n')
compound_encodings = Encodings(verbose=False)
compound_encodings.load(
os.path.join(model_folder_path, 'compound.encodings'))
config = CompoundWordConfig(
os.path.join(model_folder_path, 'compound.conf'))
self._compound_word_expander = CompoundWordExpander(
config, compound_encodings, embeddings, runtime=True)
self._compound_word_expander.load(
os.path.join(model_folder_path, 'compound.bestAcc'))
# 4. Load lemmatizer
if lemmatization:
if not os.path.isfile(
os.path.join(model_folder_path, 'lemmatizer.bestACC')):
sys.stdout.write(
'\tLemmatizer is not available on this model. \n')
else:
if self._verbose:
sys.stdout.write('\tLoading lemmatization model ...\n')
lemmatizer_encodings = Encodings(verbose=False)
lemmatizer_encodings.load(
os.path.join(model_folder_path, 'lemmatizer.encodings'))
config = LemmatizerConfig(
os.path.join(model_folder_path, 'lemmatizer.conf'))
self._lemmatizer = FSTLemmatizer(config,
lemmatizer_encodings,
embeddings,
runtime=True)
self._lemmatizer.load(
os.path.join(model_folder_path, 'lemmatizer.bestACC'))
# 5. Load taggers
if tagging or lemmatization: # we need tagging for lemmatization
if not os.path.isfile(
os.path.join(model_folder_path, 'tagger.bestUPOS')):
sys.stdout.write(
'\tTagging is not available on this model. \n')
if lemmatization:
sys.stdout.write(
'\t\tDisabling the lemmatization model due to missing tagger. \n'
)
self._lemmatizer = None
else:
if self._verbose:
sys.stdout.write('\tLoading tagger model ...\n')
tagger_encodings = Encodings(verbose=False)
tagger_encodings.load(
os.path.join(model_folder_path, 'tagger.encodings'))
config = TaggerConfig(
os.path.join(model_folder_path, 'tagger.conf'))
self._tagger = [None, None, None]
self._tagger[0] = BDRNNTagger(config,
tagger_encodings,
embeddings,
runtime=True)
self._tagger[0].load(
os.path.join(model_folder_path, 'tagger.bestUPOS'))
self._tagger[1] = BDRNNTagger(config,
tagger_encodings,
embeddings,
runtime=True)
self._tagger[1].load(
os.path.join(model_folder_path, 'tagger.bestXPOS'))
self._tagger[2] = BDRNNTagger(config,
tagger_encodings,
embeddings,
runtime=True)
self._tagger[2].load(
os.path.join(model_folder_path, 'tagger.bestATTRS'))
# 6. Load parser
if parsing:
if not os.path.isfile(
os.path.join(model_folder_path, 'parser.bestUAS')):
sys.stdout.write(
'\tParsing is not available on this model... \n')
else:
if self._verbose:
sys.stdout.write('\tLoading parser model ...\n')
parser_encodings = Encodings(verbose=False)
parser_encodings.load(
os.path.join(model_folder_path, 'parser.encodings'))
config = ParserConfig(
os.path.join(model_folder_path, 'parser.conf'))
self._parser = BDRNNParser(config,
parser_encodings,
embeddings,
runtime=True)
self._parser.load(
os.path.join(model_folder_path, 'parser.bestUAS'))
self._loaded = True
if self._verbose:
sys.stdout.write('Model loading complete.\n\n')
def __call__(self, text):
if not self._loaded:
raise Exception(
"Cube object is initialized but no model is loaded (eg.: call cube.load('en') )"
)
sequences = []
if self._tokenizer:
# split text by lines
input_lines = text.split("\n")
for input_line in input_lines:
sequences += self._tokenizer.tokenize(input_line)
if self._compound_word_expander:
sequences = self._compound_word_expander.expand_sequences(
sequences)
if self._parser:
sequences = self._parser.parse_sequences(sequences)
if self._tagger or self._lemmatizer:
import copy
new_sequences = []
for sequence in sequences:
new_sequence = copy.deepcopy(sequence)
predicted_tags_UPOS = self._tagger[0].tag(new_sequence)
predicted_tags_XPOS = self._tagger[1].tag(new_sequence)
predicted_tags_ATTRS = self._tagger[2].tag(new_sequence)
for entryIndex in range(len(sequence)):
new_sequence[entryIndex].upos = predicted_tags_UPOS[
entryIndex][0]
new_sequence[entryIndex].xpos = predicted_tags_XPOS[
entryIndex][1]
new_sequence[entryIndex].attrs = predicted_tags_ATTRS[
entryIndex][2]
new_sequences.append(new_sequence)
sequences = new_sequences
if self._lemmatizer:
sequences = self._lemmatizer.lemmatize_sequences(sequences)
return sequences
def load(self, lang_code, base_path=None):
"""
Loads the pipeline with all available models for the target language
@param lang_code: Target language code. See http://opensource.adobe.com/NLP-Cube/ for available languages and their codes
@param base_path: Base path for models. Only required for custom-trained models. Otherwise, just leave this parameter untouched to use the default model location
@return: True if loading was successful, False otherwise
"""
sys.stdout.write('Loading models for ' + lang_code + "\n")
if base_path is None:
global BASE_PATH
base_path = BASE_PATH
self.embeddings = WordEmbeddings()
self.embeddings.read_from_file(os.path.join(base_path, lang_code + "/wiki." + lang_code + ".vec"), None,
full_load=False)
if not os.path.isfile(os.path.join(base_path, lang_code + "/tokenizer-tok.bestAcc")):
sys.stdout.write(
"\tTokenization disabled. \n")
else:
self.tokenizer_enabled = True
sys.stdout.write("\tTokenization enabled.\n")
tokenizer_encodings = Encodings(verbose=False)
tokenizer_encodings.load(os.path.join(base_path, lang_code + "/tokenizer.encodings"))
from io_utils.config import TieredTokenizerConfig
from generic_networks.tokenizers import TieredTokenizer
config = TieredTokenizerConfig(os.path.join(base_path, lang_code + "/tokenizer.conf"))
tokenizer_object = TieredTokenizer(config, tokenizer_encodings, self.embeddings, runtime=True)
tokenizer_object.load(os.path.join(base_path, lang_code + "/tokenizer"))
self.models[PipelineComponents.TOKENIZER] = tokenizer_object
if not os.path.isfile(os.path.join(base_path, lang_code + "/compound.bestAcc")):
sys.stdout.write(
"\tCompound disabled. \n")
else:
self.compound_enabled = True
sys.stdout.write("\tCompound enabled.\n")
compound_encodings = Encodings(verbose=False)
compound_encodings.load(os.path.join(base_path, lang_code + "/compound.encodings"))
from io_utils.config import CompoundWordConfig
from generic_networks.token_expanders import CompoundWordExpander
config = CompoundWordConfig(os.path.join(base_path, lang_code + "/compound.conf"))
compound_object = CompoundWordExpander(config, compound_encodings, self.embeddings, runtime=True)
compound_object.load(os.path.join(base_path, lang_code + "/compound.bestAcc"))
self.models[PipelineComponents.COMPOUND] = compound_object
if not os.path.isfile(os.path.join(base_path, lang_code + "/lemmatizer.bestACC")):
sys.stdout.write(
"\tLemmatizer disabled. \n")
else:
self.lemmatizer_enabled = True
sys.stdout.write("\tLemmatizer enabled.\n")
lemmatizer_encodings = Encodings(verbose=False)
lemmatizer_encodings.load(os.path.join(base_path, lang_code + "/lemmatizer.encodings"))
from io_utils.config import LemmatizerConfig
from generic_networks.lemmatizers import FSTLemmatizer
config = LemmatizerConfig(os.path.join(base_path, lang_code + "/lemmatizer.conf"))
lemmatizer_object = FSTLemmatizer(config, lemmatizer_encodings, self.embeddings, runtime=True)
lemmatizer_object.load(os.path.join(base_path, lang_code + "/lemmatizer.bestACC"))
self.models[PipelineComponents.LEMMATIZER] = lemmatizer_object
if not os.path.isfile(os.path.join(base_path, lang_code + "/tagger.bestUPOS")):
sys.stdout.write(
"\tTagger disabled. \n")
else:
self.tagger_enabled = True
sys.stdout.write("\tTagger enabled.\n")
tagger_encodings = Encodings(verbose=False)
tagger_encodings.load(os.path.join(base_path, lang_code + "/tagger.encodings"))
from io_utils.config import TaggerConfig
from generic_networks.taggers import BDRNNTagger
config = TaggerConfig(os.path.join(base_path, lang_code + "/tagger.conf"))
tagger_upos_object = BDRNNTagger(config, tagger_encodings, self.embeddings, runtime=True)
tagger_upos_object.load(os.path.join(base_path, lang_code + "/tagger.bestUPOS"))
tagger_xpos_object = BDRNNTagger(config, tagger_encodings, self.embeddings, runtime=True)
tagger_xpos_object.load(os.path.join(base_path, tagger_encodings + "/tagger.bestXPOS"))
tagger_attrs_object = BDRNNTagger(config, tagger_encodings, self.embeddings, runtime=True)
tagger_attrs_object.load(os.path.join(base_path, lang_code + "/tagger.bestATTRS"))
self.models[PipelineComponents.TAGGER] = [tagger_upos_object, tagger_xpos_object, tagger_attrs_object]
if not os.path.isfile(os.path.join(base_path, lang_code + "/parser.bestUAS")):
sys.stdout.write(
"\tParser disabled. \n")
else:
self.parser_enabled = True
sys.stdout.write("\tParser enabled.\n")
lemmatizer_encodings = Encodings(verbose=False)
lemmatizer_encodings.load(os.path.join(base_path, lang_code + "/parser.encodings"))
from io_utils.config import ParserConfig
from generic_networks.parsers import BDRNNParser
config = ParserConfig(os.path.join(base_path, lang_code + "/parser.conf"))
parser_object = BDRNNParser(config, lemmatizer_encodings, self.embeddings, runtime=True)
parser_object.load(os.path.join(base_path, lang_code + "/parser.bestUAS"))
self.models[PipelineComponents.PARSER] = parser_object
def _load(self, lang_code):
"""
Load models on the class.
"""
sys.stdout.write('Loading models for {}\n'.format(lang_code))
path_for_language = os.path.join(self.disk_path, lang_code)
# 1. Load word embeddings.
self.embeddings = WordEmbeddings()
word_embeddings_for_language = 'wiki.{}.vec'.format(lang_code)
self.embeddings.read_from_file(os.path.join(
path_for_language, word_embeddings_for_language),
None,
full_load=False)
# 2. Load tokenizer.
if not os.path.isfile(
os.path.join(path_for_language, 'tokenizer-tok.bestAcc')):
sys.stdout.write('\tTokenization disabled. \n')
else:
self.tokenizer_enabled = True
sys.stdout.write('\tTokenization enabled.\n')
tokenizer_encodings = Encodings(verbose=False)
tokenizer_encodings.load(
os.path.join(path_for_language, 'tokenizer.encodings'))
config = TieredTokenizerConfig(
os.path.join(path_for_language, 'tokenizer.conf'))
tokenizer_object = TieredTokenizer(config,
tokenizer_encodings,
self.embeddings,
runtime=True)
tokenizer_object.load(os.path.join(path_for_language, 'tokenizer'))
self.models[PipelineComponents.TOKENIZER] = tokenizer_object
# 3. Load compound.
if not os.path.isfile(
os.path.join(path_for_language, 'compound.bestAcc')):
sys.stdout.write('\tCompound disabled. \n')
else:
self.compound_enabled = True
sys.stdout.write('\tCompound enabled.\n')
compound_encodings = Encodings(verbose=False)
compound_encodings.load(
os.path.join(path_for_language, 'compound.encodings'))
config = CompoundWordConfig(
os.path.join(path_for_language, 'compound.conf'))
compound_object = CompoundWordExpander(config,
compound_encodings,
self.embeddings,
runtime=True)
compound_object.load(
os.path.join(path_for_language, 'compound.bestAcc'))
self.models[PipelineComponents.COMPOUND] = compound_object
if not os.path.isfile(
os.path.join(path_for_language, 'lemmatizer.bestACC')):
sys.stdout.write('\tLemmatizer disabled. \n')
else:
self.lemmatizer_enabled = True
sys.stdout.write('\tLemmatizer enabled.\n')
lemmatizer_encodings = Encodings(verbose=False)
lemmatizer_encodings.load(
os.path.join(path_for_language, 'lemmatizer.encodings'))
config = LemmatizerConfig(
os.path.join(path_for_language, 'lemmatizer.conf'))
lemmatizer_object = FSTLemmatizer(config,
lemmatizer_encodings,
self.embeddings,
runtime=True)
lemmatizer_object.load(
os.path.join(path_for_language, 'lemmatizer.bestACC'))
self.models[PipelineComponents.LEMMATIZER] = lemmatizer_object
if not os.path.isfile(
os.path.join(path_for_language, 'tagger.bestUPOS')):
sys.stdout.write('\tTagger disabled. \n')
else:
self.tagger_enabled = True
sys.stdout.write('\tTagger enabled.\n')
tagger_encodings = Encodings(verbose=False)
tagger_encodings.load(
os.path.join(path_for_language, 'tagger.encodings'))
config = TaggerConfig(
os.path.join(path_for_language, 'tagger.conf'))
tagger_upos_object = BDRNNTagger(config,
tagger_encodings,
self.embeddings,
runtime=True)
tagger_upos_object.load(
os.path.join(path_for_language, 'tagger.bestUPOS'))
tagger_xpos_object = BDRNNTagger(config,
tagger_encodings,
self.embeddings,
runtime=True)
tagger_xpos_object.load(
os.path.join(path_for_language, 'tagger.bestXPOS'))
tagger_attrs_object = BDRNNTagger(config,
tagger_encodings,
self.embeddings,
runtime=True)
tagger_attrs_object.load(
os.path.join(path_for_language, 'tagger.bestATTRS'))
self.models[PipelineComponents.TAGGER] = [
tagger_upos_object, tagger_xpos_object, tagger_attrs_object
]
if not os.path.isfile(os.path.join(path_for_language,
'parser.bestUAS')):
sys.stdout.write('\tParser disabled. \n')
else:
self.parser_enabled = True
sys.stdout.write('\tParser enabled.\n')
lemmatizer_encodings = Encodings(verbose=False)
lemmatizer_encodings.load(
os.path.join(path_for_language, 'parser.encodings'))
config = ParserConfig(
os.path.join(path_for_language, 'parser.conf'))
parser_object = BDRNNParser(config,
lemmatizer_encodings,
self.embeddings,
runtime=True)
parser_object.load(
os.path.join(path_for_language, 'parser.bestUAS'))
self.models[PipelineComponents.PARSER] = parser_object
def _load(self, lang_code, version):
"""
Load models on the class.
"""
# Refresh metadata
self.metadata.read(
os.path.join(self.disk_path, lang_code + "-" + str(version),
"metadata.json"))
model_folder = os.path.join(self.disk_path,
lang_code + "-" + str(version))
embeddings_folder = os.path.join(self.disk_path, "embeddings")
embeddings_file_path = os.path.join(embeddings_folder,
self.metadata.embeddings_file_name)
#sys.stdout.write('Loading model for {}-{}\n'.format(lang_code,version))
# 1. Load word embeddings
self.embeddings = WordEmbeddings(verbose=False)
sys.stdout.write('\tLoading embeddings... \n')
self.embeddings.read_from_file(embeddings_file_path,
None,
full_load=False)
# 2. Load tokenizer
if not os.path.isfile(
os.path.join(model_folder, 'tokenizer-tok.bestAcc')):
sys.stdout.write('\tTokenization disabled. \n')
else:
self.tokenizer_enabled = True
sys.stdout.write('\tTokenization enabled.\n')
tokenizer_encodings = Encodings(verbose=False)
tokenizer_encodings.load(
os.path.join(model_folder, 'tokenizer.encodings'))
config = TieredTokenizerConfig(
os.path.join(model_folder, 'tokenizer.conf'))
tokenizer_object = TieredTokenizer(config,
tokenizer_encodings,
self.embeddings,
runtime=True)
tokenizer_object.load(os.path.join(model_folder, 'tokenizer'))
self.model[PipelineComponents.TOKENIZER] = tokenizer_object
# 3. Load compound
if not os.path.isfile(os.path.join(model_folder, 'compound.bestAcc')):
sys.stdout.write('\tCompound disabled. \n')
else:
self.compound_enabled = True
sys.stdout.write('\tCompound enabled.\n')
compound_encodings = Encodings(verbose=False)
compound_encodings.load(
os.path.join(model_folder, 'compound.encodings'))
config = CompoundWordConfig(
os.path.join(model_folder, 'compound.conf'))
compound_object = CompoundWordExpander(config,
compound_encodings,
self.embeddings,
runtime=True)
compound_object.load(os.path.join(model_folder,
'compound.bestAcc'))
self.model[PipelineComponents.COMPOUND] = compound_object
# 4. Load lemmatizer
if not os.path.isfile(os.path.join(model_folder,
'lemmatizer.bestACC')):
sys.stdout.write('\tLemmatizer disabled. \n')
else:
self.lemmatizer_enabled = True
sys.stdout.write('\tLemmatizer enabled.\n')
lemmatizer_encodings = Encodings(verbose=False)
lemmatizer_encodings.load(
os.path.join(model_folder, 'lemmatizer.encodings'))
config = LemmatizerConfig(
os.path.join(model_folder, 'lemmatizer.conf'))
lemmatizer_object = FSTLemmatizer(config,
lemmatizer_encodings,
self.embeddings,
runtime=True)
lemmatizer_object.load(
os.path.join(model_folder, 'lemmatizer.bestACC'))
self.model[PipelineComponents.LEMMATIZER] = lemmatizer_object
# 5. Load taggers
if not os.path.isfile(os.path.join(model_folder, 'tagger.bestUPOS')):
sys.stdout.write('\tTagger disabled. \n')
else:
self.tagger_enabled = True
sys.stdout.write('\tTagger enabled.\n')
tagger_encodings = Encodings(verbose=False)
tagger_encodings.load(
os.path.join(model_folder, 'tagger.encodings'))
config = TaggerConfig(os.path.join(model_folder, 'tagger.conf'))
tagger_upos_object = BDRNNTagger(config,
tagger_encodings,
self.embeddings,
runtime=True)
tagger_upos_object.load(
os.path.join(model_folder, 'tagger.bestUPOS'))
tagger_xpos_object = BDRNNTagger(config,
tagger_encodings,
self.embeddings,
runtime=True)
tagger_xpos_object.load(
os.path.join(model_folder, 'tagger.bestXPOS'))
tagger_attrs_object = BDRNNTagger(config,
tagger_encodings,
self.embeddings,
runtime=True)
tagger_attrs_object.load(
os.path.join(model_folder, 'tagger.bestATTRS'))
self.model[PipelineComponents.TAGGER] = [
tagger_upos_object, tagger_xpos_object, tagger_attrs_object
]
# 6. Load parser
if not os.path.isfile(os.path.join(model_folder, 'parser.bestUAS')):
sys.stdout.write('\tParser disabled. \n')
else:
self.parser_enabled = True
sys.stdout.write('\tParser enabled.\n')
lemmatizer_encodings = Encodings(verbose=False)
lemmatizer_encodings.load(
os.path.join(model_folder, 'parser.encodings'))
config = ParserConfig(os.path.join(model_folder, 'parser.conf'))
parser_object = BDRNNParser(config,
lemmatizer_encodings,
self.embeddings,
runtime=True)
parser_object.load(os.path.join(model_folder, 'parser.bestUAS'))
self.model[PipelineComponents.PARSER] = parser_object