def persist(self, path, persistor=None, create_unique_subfolder=True):
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
if create_unique_subfolder:
dir_name = os.path.join(path, "model_" + timestamp)
os.mkdir(dir_name)
else:
dir_name = path
data_file = os.path.join(dir_name, "training_data.json")
classifier_file, entity_extractor_file = None, None
if self.intent_classifier:
classifier_file = os.path.join(dir_name, "intent_classifier.dat")
if self.entity_extractor:
entity_extractor_file = os.path.join(dir_name,
"entity_extractor.dat")
write_training_metadata(dir_name, timestamp, data_file, self.name,
'en', classifier_file, entity_extractor_file,
self.fe_file)
with open(data_file, 'w') as f:
f.write(self.training_data.as_json(indent=2))
self.intent_classifier.save_to_disk(classifier_file, pure_model=True)
if self.entity_extractor:
self.entity_extractor.save_to_disk(entity_extractor_file,
pure_model=True)
if persistor is not None:
persistor.send_tar_to_s3(dir_name)
def persist(self, path, persistor=None):
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
dir_name = os.path.join(path, "model_" + timestamp)
os.mkdir(dir_name)
data_file = os.path.join(dir_name, "training_data.json")
classifier_file = os.path.join(dir_name, "intent_classifier.pkl")
ner_dir = os.path.join(dir_name, 'ner')
os.mkdir(ner_dir)
entity_extractor_config_file = os.path.join(ner_dir, "config.json")
entity_extractor_file = os.path.join(ner_dir, "model")
write_training_metadata(dir_name, timestamp, data_file, self.name,
self.language_name, classifier_file, ner_dir)
with open(data_file, 'w') as f:
f.write(self.training_data.as_json(indent=2))
with open(classifier_file, 'wb') as f:
cloudpickle.dump(self.intent_classifier, f)
with open(entity_extractor_config_file, 'w') as f:
json.dump(self.entity_extractor.ner.cfg, f)
self.entity_extractor.ner.model.dump(entity_extractor_file)
if persistor is not None:
persistor.send_tar_to_s3(dirname)
def persist(self, path, persistor=None, create_unique_subfolder=True):
entity_extractor_file, entity_extractor_config_file = None, None
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
if create_unique_subfolder:
dir_name = os.path.join(path, "model_" + timestamp)
os.makedirs(dir_name)
else:
dir_name = path
data_file = os.path.join(dir_name, "training_data.json")
entity_synonyms_file = os.path.join(
dir_name,
"index.json") if self.training_data.entity_synonyms else None
classifier_file, ner_dir, entity_extractor_config_file, entity_extractor_file = None, None, None, None
if self.intent_classifier:
classifier_file = os.path.join(dir_name, "intent_classifier.pkl")
if self.entity_extractor:
ner_dir = os.path.join(dir_name, 'ner')
if not os.path.exists(ner_dir):
os.mkdir(ner_dir)
entity_extractor_config_file = os.path.join(ner_dir, "config.json")
entity_extractor_file = os.path.join(ner_dir, "model")
write_training_metadata(
dir_name,
timestamp,
data_file,
SPACY_BACKEND_NAME,
self.language_name,
classifier_file,
ner_dir,
entity_synonyms_file,
should_fine_tune_spacy_ner=self.should_fine_tune_spacy_ner)
with open(data_file, 'w') as f:
f.write(self.training_data.as_json(indent=2))
if self.intent_classifier:
with open(classifier_file, 'wb') as f:
cloudpickle.dump(self.intent_classifier, f)
if self.entity_extractor:
with open(entity_extractor_config_file, 'w') as f:
json.dump(self.entity_extractor.ner.cfg, f)
self.entity_extractor.ner.model.dump(entity_extractor_file)
if self.training_data.entity_synonyms:
with open(entity_synonyms_file, 'w') as f:
json.dump(self.training_data.entity_synonyms, f)
if persistor is not None:
persistor.send_tar_to_s3(dir_name)
return dir_name
def persist(self,path):
tstamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
dirname = os.path.join(path,"model_"+tstamp)
os.mkdir(dirname)
data_file = os.path.join(dirname,"training_data.json")
classifier_file = os.path.join(dirname,"intent_classifier.dat")
entity_extractor_file = os.path.join(dirname,"entity_extractor.dat")
write_training_metadata(dirname, tstamp, data_file, self.name, 'en',
classifier_file, entity_extractor_file, self.fe_file)
with open(data_file,'w') as f:
f.write(self.training_data.as_json(indent=2))
self.intent_classifier.save_to_disk(classifier_file,pure_model=True)
self.entity_extractor.save_to_disk(entity_extractor_file,pure_model=True)
def persist(self, path, persistor=None, create_unique_subfolder=True):
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
if create_unique_subfolder:
dir_name = os.path.join(path, "model_" + timestamp)
os.mkdir(dir_name)
else:
dir_name = path
data_file = os.path.join(dir_name, "training_data.json")
classifier_file, entity_extractor_file = None, None
if self.intent_classifier:
classifier_file = os.path.join(dir_name, "intent_classifier.pkl")
if self.entity_extractor:
entity_extractor_file = os.path.join(dir_name,
"entity_extractor.dat")
entity_synonyms_file = os.path.join(
dir_name,
"index.json") if self.training_data.entity_synonyms else None
write_training_metadata(dir_name, timestamp, data_file,
MITIE_SKLEARN_BACKEND_NAME, 'en',
classifier_file, entity_extractor_file,
entity_synonyms_file, self.fe_file)
with open(data_file, 'w') as f:
f.write(self.training_data.as_json(indent=2))
if self.training_data.entity_synonyms:
with open(entity_synonyms_file, 'w') as f:
json.dump(self.training_data.entity_synonyms, f)
if self.intent_classifier:
with open(classifier_file, 'wb') as f:
cloudpickle.dump(self.intent_classifier, f)
if self.entity_extractor:
self.entity_extractor.save_to_disk(entity_extractor_file,
pure_model=True)
if persistor is not None:
persistor.send_tar_to_s3(dir_name)
return dir_name
