def __init__(self, model=None, in_bytes=False): self.f = fasttext.fasttext() if model is not None: if in_bytes: self.f.loadModel(model, len(model)) else: self.f.loadModel(model)
def __init__(self, model_path=None, args=None): self.f = fasttext.fasttext() if model_path is not None: self.f.loadModel(model_path) self._words = None self._labels = None self.set_args(args)
def tokenize(text): """Given a string of text, tokenize it and return a list of tokens""" if text.find('\n') != -1: raise ValueError( "tokenize processes one line at a time (remove \'\\n\')" ) f = fasttext.fasttext() return f.tokenize(text)
def __init__(self, model_path=None, args=None): self.f = fasttext.fasttext() if model_path is not None: self.f.loadModel(model_path) self._words = None self._labels = None if args: arg_names = ['lr', 'dim', 'ws', 'epoch', 'minCount', 'minCountLabel', 'minn', 'maxn', 'neg', 'wordNgrams', 'loss', 'bucket', 'thread', 'lrUpdateRate', 't', 'label', 'verbose', 'pretrainedVectors'] for arg_name in arg_names: setattr(self, arg_name, getattr(args, arg_name))
def __init__(self, model=None): self.f = fasttext.fasttext() if model is not None: self.f.loadModel(model)
def tokenize(text): """Given a string of text, tokenize it and return a list of tokens""" f = fasttext.fasttext() return f.tokenize(text)
def _build_args(args): args["model"] = _parse_model_string(args["model"]) args["loss"] = _parse_loss_string(args["loss"]) a = fasttext.args() for (k, v) in args.items(): setattr(a, k, v) a.output = "" # User should use save_model a.pretrainedVectors = "" # Unsupported a.saveOutput = 0 # Never use this if a.wordNgrams <= 1 and a.maxn == 0: a.bucket = 0 return a ftobj = fasttext.fasttext() def tokenize(text): """Given a string of text, tokenize it and return a list of tokens""" return ftobj.tokenize(text) """ my definitions """ model_dict = dict() label_t, value_t = '__label__true', 1 label_f, value_f = '__label__false', 0 binary_label2value = {label_t: value_t, label_f: value_f} def load_model(path):