def save(self):
"""
Save model to file.
"""
if not self.ser_path.parent.exists():
self.ser_path.parent.mkdir(mode=0o755)
save_pickle(self._estimator, self.ser_path.as_posix())
print(':: model saved to {}'.format(self.ser_path))
def save(self) -> None:
save_pickle(self.inverted_index,
self.save_path / self.inverted_index_filename)
save_pickle(self.entities_list,
self.save_path / self.entities_list_filename)
save_pickle(self.q2name, self.save_path / self.q2name_filename)
if self.q2descr_filename is not None:
save_pickle(self.q2descr, self.save_path / self.q2descr_filename)
def __init__(self, data_dir=None, *args, **kwargs):
if data_dir is None:
data_dir = paths.USR_PATH
data_dir = Path(data_dir)
if self.dict_name is None:
self.dict_name = args[0] if args else kwargs.get(
'dictionary_name', 'dictionary')
data_dir = data_dir / self.dict_name
alphabet_path = data_dir / 'alphabet.pkl'
words_path = data_dir / 'words.pkl'
words_trie_path = data_dir / 'words_trie.pkl'
if not is_done(data_dir):
print('Trying to build a dictionary in {}'.format(data_dir),
file=sys.stderr)
if data_dir.is_dir():
shutil.rmtree(data_dir)
data_dir.mkdir(parents=True)
words = self._get_source(data_dir, *args, **kwargs)
words = {self._normalize(word) for word in words}
alphabet = {c for w in words for c in w}
alphabet.remove('⟬')
alphabet.remove('⟭')
save_pickle(alphabet, alphabet_path)
save_pickle(words, words_path)
words_trie = defaultdict(set)
for word in words:
for i in range(len(word)):
words_trie[word[:i]].add(word[:i + 1])
words_trie[word] = set()
words_trie = {k: sorted(v) for k, v in words_trie.items()}
save_pickle(words_trie, words_trie_path)
mark_done(data_dir)
print('built', file=sys.stderr)
else:
print('Loading a dictionary from {}'.format(data_dir),
file=sys.stderr)
self.alphabet = load_pickle(alphabet_path)
self.words_set = load_pickle(words_path)
self.words_trie = load_pickle(words_trie_path)
def __init__(self, data_dir=None, *args, **kwargs):
if data_dir is None:
data_dir = paths.USR_PATH
data_dir = Path(data_dir)
if self.dict_name is None:
self.dict_name = args[0] if args else kwargs.get('dictionary_name', 'dictionary')
data_dir = data_dir / self.dict_name
alphabet_path = data_dir / 'alphabet.pkl'
words_path = data_dir / 'words.pkl'
words_trie_path = data_dir / 'words_trie.pkl'
if not is_done(data_dir):
print('Trying to build a dictionary in {}'.format(data_dir), file=sys.stderr)
if data_dir.is_dir():
shutil.rmtree(data_dir)
data_dir.mkdir(parents=True)
words = self._get_source(data_dir, *args, **kwargs)
words = {self._normalize(word) for word in words}
alphabet = {c for w in words for c in w}
alphabet.remove('⟬')
alphabet.remove('⟭')
save_pickle(alphabet, alphabet_path)
save_pickle(words, words_path)
words_trie = defaultdict(set)
for word in words:
for i in range(len(word)):
words_trie[word[:i]].add(word[:i+1])
words_trie[word] = set()
words_trie = {k: sorted(v) for k, v in words_trie.items()}
save_pickle(words_trie, words_trie_path)
mark_done(data_dir)
print('built', file=sys.stderr)
else:
print('Loading a dictionary from {}'.format(data_dir), file=sys.stderr)
self.alphabet = load_pickle(alphabet_path)
self.words_set = load_pickle(words_path)
self.words_trie = load_pickle(words_trie_path)
def __init__(self,
data_dir: [Path, str] = '',
*args,
dictionary_name: str = 'dictionary',
**kwargs):
data_dir = expand_path(data_dir) / dictionary_name
alphabet_path = data_dir / 'alphabet.pkl'
words_path = data_dir / 'words.pkl'
words_trie_path = data_dir / 'words_trie.pkl'
if not is_done(data_dir):
log.info('Trying to build a dictionary in {}'.format(data_dir))
if data_dir.is_dir():
shutil.rmtree(str(data_dir))
data_dir.mkdir(parents=True)
words = self._get_source(data_dir, *args, **kwargs)
words = {self._normalize(word) for word in words}
alphabet = {c for w in words for c in w}
alphabet.remove('⟬')
alphabet.remove('⟭')
save_pickle(alphabet, alphabet_path)
save_pickle(words, words_path)
words_trie = defaultdict(set)
for word in words:
for i in range(len(word)):
words_trie[word[:i]].add(word[:i + 1])
words_trie[word] = set()
words_trie = {k: sorted(v) for k, v in words_trie.items()}
save_pickle(words_trie, words_trie_path)
mark_done(data_dir)
log.info('built')
else:
log.info('Loading a dictionary from {}'.format(data_dir))
self.alphabet = load_pickle(alphabet_path)
self.words_set = load_pickle(words_path)
self.words_trie = load_pickle(words_trie_path)
def __init__(self, data_dir: [Path, str]='', *args, dictionary_name: str='dictionary', **kwargs):
data_dir = expand_path(data_dir) / dictionary_name
alphabet_path = data_dir / 'alphabet.pkl'
words_path = data_dir / 'words.pkl'
words_trie_path = data_dir / 'words_trie.pkl'
if not is_done(data_dir):
log.info('Trying to build a dictionary in {}'.format(data_dir))
if data_dir.is_dir():
shutil.rmtree(str(data_dir))
data_dir.mkdir(parents=True)
words = self._get_source(data_dir, *args, **kwargs)
words = {self._normalize(word) for word in words}
alphabet = {c for w in words for c in w}
alphabet.remove('⟬')
alphabet.remove('⟭')
save_pickle(alphabet, alphabet_path)
save_pickle(words, words_path)
words_trie = defaultdict(set)
for word in words:
for i in range(len(word)):
words_trie[word[:i]].add(word[:i+1])
words_trie[word] = set()
words_trie = {k: sorted(v) for k, v in words_trie.items()}
save_pickle(words_trie, words_trie_path)
mark_done(data_dir)
log.info('built')
else:
log.info('Loading a dictionary from {}'.format(data_dir))
self.alphabet = load_pickle(alphabet_path)
self.words_set = load_pickle(words_path)
self.words_trie = load_pickle(words_trie_path)
def save(self) -> None:
"""Save model"""
logger.info("Saving tfidf_vectorizer to {}".format(self.save_path))
path = expand_path(self.save_path)
make_all_dirs(path)
save_pickle(self.vectorizer, path)
def save(self, **kwargs) -> None:
"""Save classifier parameters"""
log.info(f"Saving model to {self.save_path}")
save_pickle(self.ec_data, self.save_path)
def save_vectorizers_data(self) -> None:
save_pickle(self.vectorizer, self.save_path / self.vectorizer_filename)
faiss.write_index(self.faiss_index, str(expand_path(self.faiss_index_filename)))
def save(self) -> None:
"""Save TF-IDF vectorizer"""
path = expand_path(self.save_path)
make_all_dirs(path)
logger.info("Saving tfidf_vectorizer to {}".format(path))
save_pickle(self.vectorizer, path)
def save(self) -> None:
logger.info("Saving classifier to {}".format(self.save_path))
save_pickle(self.clf, expand_path(self.save_path))
def save(self) -> None:
"""Save classifier parameters"""
logger.info("Saving faq_model to {}".format(self.save_path))
path = expand_path(self.save_path)
make_all_dirs(path)
save_pickle((self.x_train_features, self.y_train), path)
def save(self) -> None:
"""Save classifier parameters"""
logger.info("Saving faq_model to {}".format(self.save_path))
path = expand_path(self.save_path)
make_all_dirs(path)
save_pickle((self.x_train_features, self.y_train), path)
def save(self) -> None:
"""Save classifier parameters"""
logger.info("Saving faq_model to {}".format(self.save_path))
save_pickle((self.x_train_features, self.y_train), self.save_path)
def save(self) -> None:
"""Save classifier parameters"""
logger.info("Saving faq_model to {}".format(self.save_path))
save_pickle((self.x_train_features, self.y_train), self.save_path)
def save(self) -> None:
"""Save classifier parameters"""
log.info("Saving to {}".format(self.save_path))
path = expand_path(self.save_path)
save_pickle((self.ec_data, self.x_train_features), path)
def save(self) -> None:
logger.info("Saving tfidf_vectorizer to {}".format(self.save_path))
save_pickle(self.vectorizer, expand_path(self.save_path))
def save(self) -> None:
logger.info("Saving faq_model to {}".format(self.save_path))
save_pickle((self.x_train_features, self.y_train),
expand_path(self.save_path))
def save(self) -> None:
"""Save classifier parameters"""
logger.info("Saving faq_logreg_model to {}".format(self.save_path))
path = expand_path(self.save_path)
make_all_dirs(path)
save_pickle(self.logreg, path)
def save(self, **kwargs) -> None:
"""Save classifier parameters"""
log.info(f"Saving model to {self.save_path}")
save_pickle(self.ec_data, self.save_path)
