def load(self, path=ELIT_DEP_BIAFFINE_EN_MIXED, model_root=None, **kwargs): """Load from disk Parameters ---------- path : str path to the directory which typically contains a config.pkl file and a model.bin file Returns ------- DepParser parser itself :param **kwargs: """ path = fetch_resource(path, model_root=model_root) config = _Config.load_json(os.path.join(path, 'config.json')) config = _Config(**config) config.save_dir = path # redirect root path to what user specified vocab = ParserVocabulary.load_json(config.save_vocab_path) vocab = ParserVocabulary(vocab) self._vocab = vocab with mx.Context(mxnet_prefer_gpu()): self._parser = BiaffineParser(vocab, config.word_dims, config.tag_dims, config.dropout_emb, config.lstm_layers, config.lstm_hiddens, config.dropout_lstm_input, config.dropout_lstm_hidden, config.mlp_arc_size, config.mlp_rel_size, config.dropout_mlp, True) self._parser.load(config.save_model_path) return self
def load_from_file(cls, model_folder, context: mx.Context = None, model_root=None, **kwargs): model_folder = fetch_resource(model_folder, model_root=model_root) if context is None: context = mxnet_prefer_gpu() config_path = os.path.join(model_folder, 'config.json') config = load_json(config_path) # convert embedding str to type embeddings = [] for classpath, param in config['embeddings']: embeddings.append((str_to_type(classpath), param)) config['embeddings'] = embeddings config['tag_dictionary'] = Dictionary.from_dict(config['tag_dictionary']) with context: embeddings = StackedEmbeddings.from_list(config['embeddings']) model = SequenceTagger( hidden_size=config['hidden_size'], embeddings=embeddings, tag_dictionary=config['tag_dictionary'], tag_type=config['tag_type'], use_crf=config['use_crf'], use_rnn=config['use_rnn'], rnn_layers=config['rnn_layers']) # print(config) model.load_parameters(os.path.join(model_folder, 'model.bin'), ctx=context) if not model.use_crf: model.transitions = pickle_load(os.path.join(model_folder, 'transitions.pkl')) # type:nd.NDArray model.transitions = model.transitions.as_in_context(context) return model
def load(self, model_path: str = ELIT_SDP_BIAFFINE_EN_MIXED, model_root=None, **kwargs): parser = self._parser = BiaffineSDPParser() model_path = fetch_resource(model_path, model_root) parser.load(model_path, self.context) return self
def __init__(self, filepath: str): """ :param filepath: the path to the binary file containing a word embedding model trained by FastText (``*.bin``). """ logging.info('FastText') logging.info('- model: {}'.format(filepath)) filepath = fetch_resource(filepath) self.model = fasttext.load_model(filepath) dim = len(self.model["king"]) super().__init__(dim)
def load_language_model(cls, model_file, context: mx.Context = None, model_root=None): realpath = fetch_resource(model_file, model_root=model_root) config = LanguageModelConfig.from_dict( load_json(os.path.join( realpath, 'config.json'))) # type: LanguageModelConfig with context: model = ContextualStringModel(config.dictionary, config.is_forward_lm, config.hidden_size, config.nlayers, config.embedding_size, config.nout, config.dropout) model.load_parameters(os.path.join(realpath, 'model.bin'), ctx=context) return model
# -*- coding:utf-8 -*- # Author: hankcs # Date: 2019-10-11 15:23 import fasttext from elit.util.io import fetch_resource url = 'https://elit-models.s3-us-west-2.amazonaws.com/cc.en.300.bin.zip' filepath = fetch_resource(url) fs = fasttext.load_model(filepath) print(fs[['fakeword', 'fakeword2']])