def load(model: str = 'xlnet', pool_mode: str = 'last', **kwargs): """ Load xlnet model. Parameters ---------- model : str, optional (default='base') Model architecture supported. Allowed values: * ``'xlnet'`` - XLNET architecture from google. pool_mode : str, optional (default='last') Model logits architecture supported. Allowed values: * ``'last'`` - last of the sequence. * ``'first'`` - first of the sequence. * ``'mean'`` - mean of the sequence. * ``'attn'`` - attention of the sequence. Returns ------- result : malaya.transformers.xlnet.Model class """ model = model.lower() pool_mode = pool_mode.lower() if pool_mode not in ['last', 'first', 'mean', 'attn']: raise Exception( "pool_mode not supported, only support ['last', 'first', 'mean', 'attn']" ) path = check_file(PATH_XLNET[model]['model'], S3_PATH_XLNET[model], **kwargs) if not os.path.exists( os.path.join(PATH_XLNET[model]['directory'], 'model.ckpt')): import tarfile with tarfile.open(path['model']) as tar: tar.extractall(path=PATH_XLNET[model]['path']) vocab_model = os.path.join(PATH_XLNET[model]['directory'], 'sp10m.cased.v9.model') vocab = os.path.join(PATH_XLNET[model]['directory'], 'sp10m.cased.v9.vocab') tokenizer = SentencePieceTokenizer(vocab_file=vocab, spm_model_file=vocab_model) xlnet_config = xlnet_lib.XLNetConfig( json_path=os.path.join(PATH_XLNET[model]['directory'], 'config.json')) xlnet_checkpoint = os.path.join(PATH_XLNET[model]['directory'], 'model.ckpt') model = Model(xlnet_config, tokenizer, xlnet_checkpoint, pool_mode=pool_mode, **kwargs) model._saver.restore(model._sess, xlnet_checkpoint) return model
def load(model: str = 'xlnet', pool_mode: str = 'last', **kwargs): """ Load xlnet model. Parameters ---------- model : str, optional (default='base') Model architecture supported. Allowed values: * ``'xlnet'`` - XLNET architecture from google. pool_mode : str, optional (default='last') Model logits architecture supported. Allowed values: * ``'last'`` - last of the sequence. * ``'first'`` - first of the sequence. * ``'mean'`` - mean of the sequence. * ``'attn'`` - attention of the sequence. Returns ------- result : malaya.transformers.xlnet.Model class """ model = model.lower() pool_mode = pool_mode.lower() from malaya.path import PATH_XLNET, S3_PATH_XLNET from malaya.function import check_file if pool_mode not in ['last', 'first', 'mean', 'attn']: raise Exception( "pool_mode not supported, only support ['last', 'first', 'mean', 'attn']" ) check_file(PATH_XLNET[model]['model'], S3_PATH_XLNET[model], **kwargs) if not os.path.exists(PATH_XLNET[model]['directory'] + 'model.ckpt'): import tarfile with tarfile.open(PATH_XLNET[model]['model']['model']) as tar: tar.extractall(path=PATH_XLNET[model]['path']) import sentencepiece as spm sp_model = spm.SentencePieceProcessor() sp_model.Load(PATH_XLNET[model]['directory'] + 'sp10m.cased.v9.model') xlnet_config = xlnet_lib.XLNetConfig( json_path=PATH_XLNET[model]['directory'] + 'config.json') xlnet_checkpoint = PATH_XLNET[model]['directory'] + 'model.ckpt' model = Model(xlnet_config, sp_model, xlnet_checkpoint, pool_mode=pool_mode, **kwargs) model._saver.restore(model._sess, xlnet_checkpoint) return model