def save(self): """ Save model to file. """ if not self.ser_path.parent.exists(): self.ser_path.parent.mkdir(mode=0o755) save_pickle(self._estimator, self.ser_path.as_posix()) print(':: model saved to {}'.format(self.ser_path))
def save(self) -> None: save_pickle(self.inverted_index, self.save_path / self.inverted_index_filename) save_pickle(self.entities_list, self.save_path / self.entities_list_filename) save_pickle(self.q2name, self.save_path / self.q2name_filename) if self.q2descr_filename is not None: save_pickle(self.q2descr, self.save_path / self.q2descr_filename)
def __init__(self, data_dir=None, *args, **kwargs): if data_dir is None: data_dir = paths.USR_PATH data_dir = Path(data_dir) if self.dict_name is None: self.dict_name = args[0] if args else kwargs.get( 'dictionary_name', 'dictionary') data_dir = data_dir / self.dict_name alphabet_path = data_dir / 'alphabet.pkl' words_path = data_dir / 'words.pkl' words_trie_path = data_dir / 'words_trie.pkl' if not is_done(data_dir): print('Trying to build a dictionary in {}'.format(data_dir), file=sys.stderr) if data_dir.is_dir(): shutil.rmtree(data_dir) data_dir.mkdir(parents=True) words = self._get_source(data_dir, *args, **kwargs) words = {self._normalize(word) for word in words} alphabet = {c for w in words for c in w} alphabet.remove('⟬') alphabet.remove('⟭') save_pickle(alphabet, alphabet_path) save_pickle(words, words_path) words_trie = defaultdict(set) for word in words: for i in range(len(word)): words_trie[word[:i]].add(word[:i + 1]) words_trie[word] = set() words_trie = {k: sorted(v) for k, v in words_trie.items()} save_pickle(words_trie, words_trie_path) mark_done(data_dir) print('built', file=sys.stderr) else: print('Loading a dictionary from {}'.format(data_dir), file=sys.stderr) self.alphabet = load_pickle(alphabet_path) self.words_set = load_pickle(words_path) self.words_trie = load_pickle(words_trie_path)
def __init__(self, data_dir=None, *args, **kwargs): if data_dir is None: data_dir = paths.USR_PATH data_dir = Path(data_dir) if self.dict_name is None: self.dict_name = args[0] if args else kwargs.get('dictionary_name', 'dictionary') data_dir = data_dir / self.dict_name alphabet_path = data_dir / 'alphabet.pkl' words_path = data_dir / 'words.pkl' words_trie_path = data_dir / 'words_trie.pkl' if not is_done(data_dir): print('Trying to build a dictionary in {}'.format(data_dir), file=sys.stderr) if data_dir.is_dir(): shutil.rmtree(data_dir) data_dir.mkdir(parents=True) words = self._get_source(data_dir, *args, **kwargs) words = {self._normalize(word) for word in words} alphabet = {c for w in words for c in w} alphabet.remove('⟬') alphabet.remove('⟭') save_pickle(alphabet, alphabet_path) save_pickle(words, words_path) words_trie = defaultdict(set) for word in words: for i in range(len(word)): words_trie[word[:i]].add(word[:i+1]) words_trie[word] = set() words_trie = {k: sorted(v) for k, v in words_trie.items()} save_pickle(words_trie, words_trie_path) mark_done(data_dir) print('built', file=sys.stderr) else: print('Loading a dictionary from {}'.format(data_dir), file=sys.stderr) self.alphabet = load_pickle(alphabet_path) self.words_set = load_pickle(words_path) self.words_trie = load_pickle(words_trie_path)
def __init__(self, data_dir: [Path, str] = '', *args, dictionary_name: str = 'dictionary', **kwargs): data_dir = expand_path(data_dir) / dictionary_name alphabet_path = data_dir / 'alphabet.pkl' words_path = data_dir / 'words.pkl' words_trie_path = data_dir / 'words_trie.pkl' if not is_done(data_dir): log.info('Trying to build a dictionary in {}'.format(data_dir)) if data_dir.is_dir(): shutil.rmtree(str(data_dir)) data_dir.mkdir(parents=True) words = self._get_source(data_dir, *args, **kwargs) words = {self._normalize(word) for word in words} alphabet = {c for w in words for c in w} alphabet.remove('⟬') alphabet.remove('⟭') save_pickle(alphabet, alphabet_path) save_pickle(words, words_path) words_trie = defaultdict(set) for word in words: for i in range(len(word)): words_trie[word[:i]].add(word[:i + 1]) words_trie[word] = set() words_trie = {k: sorted(v) for k, v in words_trie.items()} save_pickle(words_trie, words_trie_path) mark_done(data_dir) log.info('built') else: log.info('Loading a dictionary from {}'.format(data_dir)) self.alphabet = load_pickle(alphabet_path) self.words_set = load_pickle(words_path) self.words_trie = load_pickle(words_trie_path)
def __init__(self, data_dir: [Path, str]='', *args, dictionary_name: str='dictionary', **kwargs): data_dir = expand_path(data_dir) / dictionary_name alphabet_path = data_dir / 'alphabet.pkl' words_path = data_dir / 'words.pkl' words_trie_path = data_dir / 'words_trie.pkl' if not is_done(data_dir): log.info('Trying to build a dictionary in {}'.format(data_dir)) if data_dir.is_dir(): shutil.rmtree(str(data_dir)) data_dir.mkdir(parents=True) words = self._get_source(data_dir, *args, **kwargs) words = {self._normalize(word) for word in words} alphabet = {c for w in words for c in w} alphabet.remove('⟬') alphabet.remove('⟭') save_pickle(alphabet, alphabet_path) save_pickle(words, words_path) words_trie = defaultdict(set) for word in words: for i in range(len(word)): words_trie[word[:i]].add(word[:i+1]) words_trie[word] = set() words_trie = {k: sorted(v) for k, v in words_trie.items()} save_pickle(words_trie, words_trie_path) mark_done(data_dir) log.info('built') else: log.info('Loading a dictionary from {}'.format(data_dir)) self.alphabet = load_pickle(alphabet_path) self.words_set = load_pickle(words_path) self.words_trie = load_pickle(words_trie_path)
def save(self) -> None: """Save model""" logger.info("Saving tfidf_vectorizer to {}".format(self.save_path)) path = expand_path(self.save_path) make_all_dirs(path) save_pickle(self.vectorizer, path)
def save(self, **kwargs) -> None: """Save classifier parameters""" log.info(f"Saving model to {self.save_path}") save_pickle(self.ec_data, self.save_path)
def save_vectorizers_data(self) -> None: save_pickle(self.vectorizer, self.save_path / self.vectorizer_filename) faiss.write_index(self.faiss_index, str(expand_path(self.faiss_index_filename)))
def save(self) -> None: """Save TF-IDF vectorizer""" path = expand_path(self.save_path) make_all_dirs(path) logger.info("Saving tfidf_vectorizer to {}".format(path)) save_pickle(self.vectorizer, path)
def save(self) -> None: logger.info("Saving classifier to {}".format(self.save_path)) save_pickle(self.clf, expand_path(self.save_path))
def save(self) -> None: """Save classifier parameters""" logger.info("Saving faq_model to {}".format(self.save_path)) path = expand_path(self.save_path) make_all_dirs(path) save_pickle((self.x_train_features, self.y_train), path)
def save(self) -> None: """Save classifier parameters""" logger.info("Saving faq_model to {}".format(self.save_path)) path = expand_path(self.save_path) make_all_dirs(path) save_pickle((self.x_train_features, self.y_train), path)
def save(self) -> None: """Save classifier parameters""" logger.info("Saving faq_model to {}".format(self.save_path)) save_pickle((self.x_train_features, self.y_train), self.save_path)
def save(self) -> None: """Save classifier parameters""" logger.info("Saving faq_model to {}".format(self.save_path)) save_pickle((self.x_train_features, self.y_train), self.save_path)
def save(self) -> None: """Save classifier parameters""" log.info("Saving to {}".format(self.save_path)) path = expand_path(self.save_path) save_pickle((self.ec_data, self.x_train_features), path)
def save(self) -> None: logger.info("Saving tfidf_vectorizer to {}".format(self.save_path)) save_pickle(self.vectorizer, expand_path(self.save_path))
def save(self) -> None: logger.info("Saving faq_model to {}".format(self.save_path)) save_pickle((self.x_train_features, self.y_train), expand_path(self.save_path))
def save(self) -> None: """Save classifier parameters""" logger.info("Saving faq_logreg_model to {}".format(self.save_path)) path = expand_path(self.save_path) make_all_dirs(path) save_pickle(self.logreg, path)
def save(self, **kwargs) -> None: """Save classifier parameters""" log.info(f"Saving model to {self.save_path}") save_pickle(self.ec_data, self.save_path)