def load(self, path=ELIT_DEP_BIAFFINE_EN_MIXED, model_root=None, **kwargs):
"""Load from disk
Parameters
----------
path : str
path to the directory which typically contains a config.pkl file and a model.bin file
Returns
-------
DepParser
parser itself
:param **kwargs:
"""
path = fetch_resource(path, model_root=model_root)
config = _Config.load_json(os.path.join(path, 'config.json'))
config = _Config(**config)
config.save_dir = path # redirect root path to what user specified
vocab = ParserVocabulary.load_json(config.save_vocab_path)
vocab = ParserVocabulary(vocab)
self._vocab = vocab
with mx.Context(mxnet_prefer_gpu()):
self._parser = BiaffineParser(vocab, config.word_dims, config.tag_dims, config.dropout_emb,
config.lstm_layers,
config.lstm_hiddens, config.dropout_lstm_input, config.dropout_lstm_hidden,
config.mlp_arc_size,
config.mlp_rel_size, config.dropout_mlp, True)
self._parser.load(config.save_model_path)
return self
def load_from_file(cls, model_folder, context: mx.Context = None, model_root=None, **kwargs):
model_folder = fetch_resource(model_folder, model_root=model_root)
if context is None:
context = mxnet_prefer_gpu()
config_path = os.path.join(model_folder, 'config.json')
config = load_json(config_path)
# convert embedding str to type
embeddings = []
for classpath, param in config['embeddings']:
embeddings.append((str_to_type(classpath), param))
config['embeddings'] = embeddings
config['tag_dictionary'] = Dictionary.from_dict(config['tag_dictionary'])
with context:
embeddings = StackedEmbeddings.from_list(config['embeddings'])
model = SequenceTagger(
hidden_size=config['hidden_size'],
embeddings=embeddings,
tag_dictionary=config['tag_dictionary'],
tag_type=config['tag_type'],
use_crf=config['use_crf'],
use_rnn=config['use_rnn'],
rnn_layers=config['rnn_layers'])
# print(config)
model.load_parameters(os.path.join(model_folder, 'model.bin'), ctx=context)
if not model.use_crf:
model.transitions = pickle_load(os.path.join(model_folder, 'transitions.pkl')) # type:nd.NDArray
model.transitions = model.transitions.as_in_context(context)
return model
def load(self,
model_path: str = ELIT_SDP_BIAFFINE_EN_MIXED,
model_root=None,
**kwargs):
parser = self._parser = BiaffineSDPParser()
model_path = fetch_resource(model_path, model_root)
parser.load(model_path, self.context)
return self
def __init__(self, filepath: str):
"""
:param filepath: the path to the binary file containing a word embedding model trained by FastText (``*.bin``).
"""
logging.info('FastText')
logging.info('- model: {}'.format(filepath))
filepath = fetch_resource(filepath)
self.model = fasttext.load_model(filepath)
dim = len(self.model["king"])
super().__init__(dim)
def load_language_model(cls,
model_file,
context: mx.Context = None,
model_root=None):
realpath = fetch_resource(model_file, model_root=model_root)
config = LanguageModelConfig.from_dict(
load_json(os.path.join(
realpath, 'config.json'))) # type: LanguageModelConfig
with context:
model = ContextualStringModel(config.dictionary,
config.is_forward_lm,
config.hidden_size, config.nlayers,
config.embedding_size, config.nout,
config.dropout)
model.load_parameters(os.path.join(realpath, 'model.bin'), ctx=context)
return model
# -*- coding:utf-8 -*- # Author: hankcs # Date: 2019-10-11 15:23 import fasttext from elit.util.io import fetch_resource url = 'https://elit-models.s3-us-west-2.amazonaws.com/cc.en.300.bin.zip' filepath = fetch_resource(url) fs = fasttext.load_model(filepath) print(fs[['fakeword', 'fakeword2']])