def save(self, file_path):
save_path = Path(file_path)
mkdir(save_path)
param_path = Path(save_path.joinpath("params.json"))
with open(param_path, "w") as fp:
fp.write(json.dumps(self.params))
word_index_path = Path(save_path.joinpath("word2idx.json"))
with open(word_index_path, "w") as fp:
fp.write(json.dumps(self.word2idx))
tag_index_path = Path(save_path.joinpath("tag2idx.json"))
with open(tag_index_path, "w") as fp:
fp.write(json.dumps(self.tag2idx))
if self.params["use_chars"]:
char_index_path = Path(save_path.joinpath("char2idx.json"))
with open(char_index_path, "w") as fp:
fp.write(json.dumps(self.char2idx))
if self.params["use_pos_tags"]:
pos_index_path = Path(save_path.joinpath("pos2idx.json"))
with open(pos_index_path, "w") as fp:
fp.write(json.dumps(self.pos2idx))
if self.params["use_word_emb"]:
word_emb_path = Path(save_path.joinpath("word_emb.csv"))
np.savetxt(fname=str(word_emb_path),
X=self.word_vectors,
delimiter=",")
def save(self, file_path):
""" Saves models to the local disk, provided a file path.
"""
save_path = Path(file_path)
mkdir(save_path)
for _, model in self.pipeline.steps:
model.save(save_path.joinpath(model.name))
def save(self, file_path):
save_path = Path(file_path)
mkdir(save_path)
model_save_path = save_path.joinpath("word2vec.model")
config_save_path = save_path.joinpath("word2vec.config")
self.model.save(str(model_save_path))
self.config.save(config_save_path)
def save(self, file_path):
""" Saves a model to the local disk, provided a file path. """
save_path = Path(file_path)
mkdir(save_path)
model_save_path = save_path.joinpath("CRF.model")
config_save_path = save_path.joinpath("CRF.config")
joblib.dump(self.crf, model_save_path)
self.config.save(config_save_path)
def save(self, file_path):
""" Saves an ensemble of models to the local disk, provided a
file path.
"""
save_path = os.path.join(file_path, "Ensemble_NER")
mkdir(save_path)
for model in self.models:
model.save(save_path)
def save(self, file_path):
""" Saves an ensemble of models to the local disk, provided a
file path.
"""
save_path = Path(file_path)
mkdir(save_path)
for model in self.models:
model.save(save_path.joinpath(model.name))
def save(self, file_path):
save_path = Path(file_path)
mkdir(save_path)
words_path = save_path.joinpath("words.dict")
# save dictionaries
# we don't save examples for now
joblib.dump(self.word_vectorizer, words_path)
def save(self, file_path):
""" Saves a model to the local disk, provided a file path. """
save_path = Path(file_path)
mkdir(save_path)
model_save_path = save_path.joinpath("RF.model")
config_save_path = save_path.joinpath("RF.config")
joblib.dump(self.rf, model_save_path)
self.feature_extractor.save(save_path.joinpath(self.feature_extractor.name))
self.config.save(config_save_path)
def test_mkdir():
directory1 = "data"
directory2 = "data/foo"
mkdir(directory1)
mkdir(directory2)
assert_true(os.path.exists(directory1))
assert_true(os.path.exists(directory2))
def save(self, file_path):
save_path = Path(file_path)
mkdir(save_path)
# save config
config_save_path = save_path.joinpath("context.config")
self.config.save(config_save_path)
# save dictionaries
for i in self.encoders:
words_path = save_path.joinpath("f{}.dict".format(i))
joblib.dump(self.encoders[i], words_path)
def save(self, file_path):
""" Saves a model to the local disk, provided a file path. """
save_path = Path(file_path)
mkdir(save_path)
model_save_path = save_path.joinpath("BiLSTM.model")
mkdir(model_save_path)
config_save_path = save_path.joinpath("BiLSTM.config")
weights_save_path = model_save_path.joinpath("weights.h5")
params_save_path = model_save_path.joinpath("params.json")
preproc_save_path = model_save_path.joinpath("preprocessor.pickle")
self.model.save(weights_save_path, params_save_path, preproc_save_path)
self.config.save(config_save_path)
def test_rmdir():
directory1 = "data"
directory2 = "data/foo"
mkdir(directory1)
mkdir(directory2)
rmdir(directory2)
rmdir(directory1)
assert_false(os.path.exists(directory1))
assert_false(os.path.exists(directory2))
def save(self, file_path):
save_path = Path(file_path)
mkdir(save_path)
words_path = save_path.joinpath("words.dict")
labels_path = save_path.joinpath("labels.dict")
pos_path = save_path.joinpath("pos.dict")
dep_path = save_path.joinpath("dep.dict")
# save dictionaries
# we don't save examples for now
joblib.dump(self.word_vectorizer, words_path)
joblib.dump(self.label_vectorizer, labels_path)
joblib.dump(self.pos_vectorizer, pos_path)
joblib.dump(self.dep_vectorizer, dep_path)
def save(self, file_path):
""" Saves a model to the local disk, provided a file path. """
save_path = Path(file_path)
mkdir(save_path)
model_save_path = save_path.joinpath("KerasNER.model")
config_save_path = save_path.joinpath("KerasNER.config")
arch_save_path = save_path.joinpath("KerasNER.json")
encoder_save_path = save_path.joinpath("encoder")
with open(arch_save_path, 'w') as f:
params = self.model.to_json()
json.dump(json.loads(params), f, sort_keys=True, indent=4)
self.model.save_weights(model_save_path)
self.config.save(config_save_path)
self.p.save(encoder_save_path)
def save(self, file_path):
""" Saves a model to the local disk, provided a file path. """
save_path = os.path.join(file_path, "CRF_NER")
model_filename = "CRF.model"
model_save_path = os.path.join(save_path, model_filename)
metadata_filename = "CRF_metadata.json"
metadata_save_path = os.path.join(save_path, metadata_filename)
mkdir(save_path)
joblib.dump(self.crf, model_save_path)
with open(metadata_save_path, "w") as fp:
fp.write(json.dumps({"entity_label": self.entity_label}))
def save(self, file_path):
""" Saves a model to the local disk, provided a file path. """
save_path = Path(file_path)
mkdir(save_path)
model_save_path = save_path.joinpath("KerasNER.model")
config_save_path = save_path.joinpath("KerasNER.config")
arch_save_path = save_path.joinpath("KerasNER.json")
encoder_save_path = save_path.joinpath("encoder")
if self.config.get_parameter("use_crf"):
save_load_utils.save_all_weights(self.model, str(model_save_path))
else:
self.model.save(str(model_save_path))
self.config.save(config_save_path)
# human-readable model architecture in json
with open(arch_save_path, "w") as wf:
wf.write(self.model.to_json())
self.encoder.save(encoder_save_path)
def save(self, file_path):
""" Saves a model to the local disk, provided a file path. """
save_path = os.path.join(file_path, "BiLSTM_NER")
model_filename = "BiLSTM.model"
model_save_path = os.path.join(save_path, model_filename)
metadata_filename = "BiLSTM_metadata.json"
metadata_save_path = os.path.join(save_path, metadata_filename)
mkdir(save_path)
mkdir(model_save_path)
self.model.save(model_save_path)
with open(metadata_save_path, "w") as fp:
fp.write(
json.dumps({
"entity_label": self.entity_label,
"label_map": self._label_map
}))
def save(self, file_path):
""" Saves a model to the local disk, provided a file path. """
mkdir(file_path)
self.svm.save(file_path.joinpath(self.svm.name))
def __init__(self, path_to_folder):
self.path_to_folder = path_to_folder
mkdir(self.path_to_folder)
def save(self, file_path):
save_path = Path(file_path)
mkdir(save_path)
self.word_vectorizer.save(save_path)
self.pos_vectorizer.save(save_path)
self.char_vectorizer.save(save_path)
def __init__(self, path_to_files):
path_to_files = Path(path_to_files)
mkdir(path_to_files)
self.path = path_to_files
