def persist(self, path, persistor=None, create_unique_subfolder=True): timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') if create_unique_subfolder: dir_name = os.path.join(path, "model_" + timestamp) os.mkdir(dir_name) else: dir_name = path data_file = os.path.join(dir_name, "training_data.json") classifier_file, entity_extractor_file = None, None if self.intent_classifier: classifier_file = os.path.join(dir_name, "intent_classifier.dat") if self.entity_extractor: entity_extractor_file = os.path.join(dir_name, "entity_extractor.dat") write_training_metadata(dir_name, timestamp, data_file, self.name, 'en', classifier_file, entity_extractor_file, self.fe_file) with open(data_file, 'w') as f: f.write(self.training_data.as_json(indent=2)) self.intent_classifier.save_to_disk(classifier_file, pure_model=True) if self.entity_extractor: self.entity_extractor.save_to_disk(entity_extractor_file, pure_model=True) if persistor is not None: persistor.send_tar_to_s3(dir_name)
def persist(self, path, persistor=None): timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') dir_name = os.path.join(path, "model_" + timestamp) os.mkdir(dir_name) data_file = os.path.join(dir_name, "training_data.json") classifier_file = os.path.join(dir_name, "intent_classifier.pkl") ner_dir = os.path.join(dir_name, 'ner') os.mkdir(ner_dir) entity_extractor_config_file = os.path.join(ner_dir, "config.json") entity_extractor_file = os.path.join(ner_dir, "model") write_training_metadata(dir_name, timestamp, data_file, self.name, self.language_name, classifier_file, ner_dir) with open(data_file, 'w') as f: f.write(self.training_data.as_json(indent=2)) with open(classifier_file, 'wb') as f: cloudpickle.dump(self.intent_classifier, f) with open(entity_extractor_config_file, 'w') as f: json.dump(self.entity_extractor.ner.cfg, f) self.entity_extractor.ner.model.dump(entity_extractor_file) if persistor is not None: persistor.send_tar_to_s3(dirname)
def persist(self, path, persistor=None, create_unique_subfolder=True): entity_extractor_file, entity_extractor_config_file = None, None timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') if create_unique_subfolder: dir_name = os.path.join(path, "model_" + timestamp) os.makedirs(dir_name) else: dir_name = path data_file = os.path.join(dir_name, "training_data.json") entity_synonyms_file = os.path.join( dir_name, "index.json") if self.training_data.entity_synonyms else None classifier_file, ner_dir, entity_extractor_config_file, entity_extractor_file = None, None, None, None if self.intent_classifier: classifier_file = os.path.join(dir_name, "intent_classifier.pkl") if self.entity_extractor: ner_dir = os.path.join(dir_name, 'ner') if not os.path.exists(ner_dir): os.mkdir(ner_dir) entity_extractor_config_file = os.path.join(ner_dir, "config.json") entity_extractor_file = os.path.join(ner_dir, "model") write_training_metadata( dir_name, timestamp, data_file, SPACY_BACKEND_NAME, self.language_name, classifier_file, ner_dir, entity_synonyms_file, should_fine_tune_spacy_ner=self.should_fine_tune_spacy_ner) with open(data_file, 'w') as f: f.write(self.training_data.as_json(indent=2)) if self.intent_classifier: with open(classifier_file, 'wb') as f: cloudpickle.dump(self.intent_classifier, f) if self.entity_extractor: with open(entity_extractor_config_file, 'w') as f: json.dump(self.entity_extractor.ner.cfg, f) self.entity_extractor.ner.model.dump(entity_extractor_file) if self.training_data.entity_synonyms: with open(entity_synonyms_file, 'w') as f: json.dump(self.training_data.entity_synonyms, f) if persistor is not None: persistor.send_tar_to_s3(dir_name) return dir_name
def persist(self,path): tstamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') dirname = os.path.join(path,"model_"+tstamp) os.mkdir(dirname) data_file = os.path.join(dirname,"training_data.json") classifier_file = os.path.join(dirname,"intent_classifier.dat") entity_extractor_file = os.path.join(dirname,"entity_extractor.dat") write_training_metadata(dirname, tstamp, data_file, self.name, 'en', classifier_file, entity_extractor_file, self.fe_file) with open(data_file,'w') as f: f.write(self.training_data.as_json(indent=2)) self.intent_classifier.save_to_disk(classifier_file,pure_model=True) self.entity_extractor.save_to_disk(entity_extractor_file,pure_model=True)
def persist(self, path, persistor=None, create_unique_subfolder=True): timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') if create_unique_subfolder: dir_name = os.path.join(path, "model_" + timestamp) os.mkdir(dir_name) else: dir_name = path data_file = os.path.join(dir_name, "training_data.json") classifier_file, entity_extractor_file = None, None if self.intent_classifier: classifier_file = os.path.join(dir_name, "intent_classifier.pkl") if self.entity_extractor: entity_extractor_file = os.path.join(dir_name, "entity_extractor.dat") entity_synonyms_file = os.path.join( dir_name, "index.json") if self.training_data.entity_synonyms else None write_training_metadata(dir_name, timestamp, data_file, MITIE_SKLEARN_BACKEND_NAME, 'en', classifier_file, entity_extractor_file, entity_synonyms_file, self.fe_file) with open(data_file, 'w') as f: f.write(self.training_data.as_json(indent=2)) if self.training_data.entity_synonyms: with open(entity_synonyms_file, 'w') as f: json.dump(self.training_data.entity_synonyms, f) if self.intent_classifier: with open(classifier_file, 'wb') as f: cloudpickle.dump(self.intent_classifier, f) if self.entity_extractor: self.entity_extractor.save_to_disk(entity_extractor_file, pure_model=True) if persistor is not None: persistor.send_tar_to_s3(dir_name) return dir_name