def __init__(self, model=None, in_bytes=False):
self.f = fasttext.fasttext()
if model is not None:
if in_bytes:
self.f.loadModel(model, len(model))
else:
self.f.loadModel(model)
def __init__(self, model_path=None, args=None):
self.f = fasttext.fasttext()
if model_path is not None:
self.f.loadModel(model_path)
self._words = None
self._labels = None
self.set_args(args)
def tokenize(text):
"""Given a string of text, tokenize it and return a list of tokens"""
if text.find('\n') != -1:
raise ValueError(
"tokenize processes one line at a time (remove \'\\n\')"
)
f = fasttext.fasttext()
return f.tokenize(text)
def __init__(self, model_path=None, args=None):
self.f = fasttext.fasttext()
if model_path is not None:
self.f.loadModel(model_path)
self._words = None
self._labels = None
if args:
arg_names = ['lr', 'dim', 'ws', 'epoch', 'minCount',
'minCountLabel', 'minn', 'maxn', 'neg', 'wordNgrams',
'loss', 'bucket', 'thread', 'lrUpdateRate', 't',
'label', 'verbose', 'pretrainedVectors']
for arg_name in arg_names:
setattr(self, arg_name, getattr(args, arg_name))
def __init__(self, model=None):
self.f = fasttext.fasttext()
if model is not None:
self.f.loadModel(model)
def tokenize(text):
"""Given a string of text, tokenize it and return a list of tokens"""
f = fasttext.fasttext()
return f.tokenize(text)
def __init__(self, model=None):
self.f = fasttext.fasttext()
if model is not None:
self.f.loadModel(model)
def tokenize(text):
"""Given a string of text, tokenize it and return a list of tokens"""
f = fasttext.fasttext()
return f.tokenize(text)
def _build_args(args):
args["model"] = _parse_model_string(args["model"])
args["loss"] = _parse_loss_string(args["loss"])
a = fasttext.args()
for (k, v) in args.items():
setattr(a, k, v)
a.output = "" # User should use save_model
a.pretrainedVectors = "" # Unsupported
a.saveOutput = 0 # Never use this
if a.wordNgrams <= 1 and a.maxn == 0:
a.bucket = 0
return a
ftobj = fasttext.fasttext()
def tokenize(text):
"""Given a string of text, tokenize it and return a list of tokens"""
return ftobj.tokenize(text)
""" my definitions """
model_dict = dict()
label_t, value_t = '__label__true', 1
label_f, value_f = '__label__false', 0
binary_label2value = {label_t: value_t, label_f: value_f}
def load_model(path):
