def load(self, dirpath): """ Loads a trained model from local disk, given the dirpath Parameters ---------- dirpath : str a directory where model artifacts are saved. Returns ------- self """ if not os.path.exists(dirpath): raise ValueError("Model directory not found: {:s}".format(dirpath)) weights_file = os.path.join(dirpath, "weights.h5") params_file = os.path.join(dirpath, "params.json") preprocessor_file = os.path.join(dirpath, "preprocessor.pkl") if not (os.path.exists(weights_file) or os.path.exists(params_file) or os.path.exists(preprocessor_file)): raise ValueError("Model files may be corrupted, exiting") self.model_ = load_model(weights_file, params_file) self.preprocessor_ = IndexTransformer.load(preprocessor_file) self.tagger_ = Tagger(self.model_, preprocessor=self.preprocessor_) return self
def predict_with_folds(swa): test = pd.read_csv(config.data_folder + "test.csv", converters={"pos": literal_eval}) x_test = [x.split() for x in test['sentence'].tolist()] p = IndexTransformer(use_char=True) p = p.load('../models/best_transform.it') lengths = map(len, x_test) x_test = p.transform(x_test) fold_result = [] for n_fold in range(config.nfolds): path = '../models/best_model_' + str(n_fold) if swa: path += '_swa' model = load_model(path + '.h5', custom_objects={ 'CRF': CRF, 'RAdam': RAdam, 'crf_loss': crf_loss, 'crf_viterbi_accuracy': crf_viterbi_accuracy }) y_pred = model.predict(x_test, verbose=True) fold_result.append(y_pred) final_pred = np.mean(fold_result, axis=0) y_pred = p.inverse_transform(final_pred, lengths) build_submission(y_pred, 'fold')
def load_and_predict(): test = pd.read_csv(config.data_folder + "test.csv", converters={"pos": literal_eval}) x_test = [x.split() for x in test['sentence'].tolist()] p = IndexTransformer(use_char=True) p = p.load('../models/best_transform.it') model = BiLSTMCRF(char_vocab_size=p.char_vocab_size, word_vocab_size=p.word_vocab_size, num_labels=p.label_size, word_embedding_dim=300, char_embedding_dim=100, word_lstm_size=100, char_lstm_size=50, fc_dim=100, dropout=0.5, embeddings=None, use_char=True, use_crf=True) model, loss = model.build() model.load_weights('../models/' + 'best_model.h5') predict(model, p, x_test)
def load(cls, weights_file, params_file, preprocessor_file): self = cls() self.p = IndexTransformer.load(preprocessor_file) self.model = load_model(weights_file, params_file) # Added by Sonvx on Jan 14, 2021: fix issue ("<tensor> is not an element of this graph." when loading model) self.model._make_predict_function() return self
def __init__(self, process_proper_nouns=False): super().__init__(process_proper_nouns) model = load_model(os.path.join(ELMO_TAGGER_PATH, 'weights.h5'), os.path.join(ELMO_TAGGER_PATH, 'params.json')) it = IndexTransformer.load( os.path.join(ELMO_TAGGER_PATH, 'preprocessor.pkl')) self.pos_tagger = Tagger(model, preprocessor=it, tokenizer=wordpunct_tokenize)
def main(args): print('Loading objects...') model = BiLSTMCRF.load(args.weights_file, args.params_file) it = IndexTransformer.load(args.preprocessor_file) tagger = Tagger(model, preprocessor=it) print('Tagging a sentence...') res = tagger.analyze(args.sent) pprint(res)
def setUpClass(cls): weights_file = os.path.join(SAVE_ROOT, 'weights.h5') params_file = os.path.join(SAVE_ROOT, 'params.json') preprocessor_file = os.path.join(SAVE_ROOT, 'preprocessor.pickle') # Load preprocessor p = IndexTransformer.load(preprocessor_file) # Load the model. model = load_model(weights_file, params_file) # Build a tagger cls.tagger = anago.Tagger(model, preprocessor=p) cls.sent = 'President Obama is speaking at the White House.'
def test_save_and_load(self): it = IndexTransformer(lower=False) x1, y1 = it.fit_transform(self.x, self.y) x1_word, x1_char, x1_length = x1 self.assertFalse(os.path.exists(self.preprocessor_file)) it.save(self.preprocessor_file) self.assertTrue(os.path.exists(self.preprocessor_file)) it = IndexTransformer.load(self.preprocessor_file) x2, y2 = it.transform(self.x, self.y) x2_word, x2_char, x2_length = x2 np.testing.assert_array_equal(x1_word, x2_word) np.testing.assert_array_equal(x1_char, x2_char) np.testing.assert_array_equal(y1, y2)
def evaluate(swa): train = pd.read_csv(config.data_folder + "train.csv", converters={"pos": literal_eval, "tag": literal_eval}) x_train = [x.split() for x in train['sentence'].tolist()] y_train = train['tag'].tolist() p = IndexTransformer(use_char=True) p = p.load('../models/best_transform.it') oof_data = [] oof_data_pred = [] skf = KFold(n_splits=config.nfolds, random_state=config.seed, shuffle=True) for n_fold, (train_indices, val_indices) in enumerate(skf.split(x_train)): x_val = list(np.array(x_train)[val_indices]) y_val = list(np.array(y_train)[val_indices]) print(y_val[:5]) oof_data.extend([x for line in y_val for x in line]) print(oof_data[:5]) lengths = map(len, x_val) x_val = p.transform(x_val) path = '../models/best_model_' + str(n_fold) if swa: path += '_swa' model = load_model(path + '.h5', custom_objects={'CRF': CRF, 'RAdam': RAdam, 'crf_loss' : crf_loss, 'crf_viterbi_accuracy': crf_viterbi_accuracy}) # model.load_weights('../models/best_model_' + str(n_fold) + '.h5') y_pred = model.predict(x_val, verbose=True) print(y_pred[:5]) y_pred = p.inverse_transform(y_pred, lengths) print(y_pred[:5]) oof_data_pred.extend([pred for line in y_pred for pred in line]) print(oof_data_pred[:5]) bacc = balanced_accuracy_score(oof_data,oof_data_pred) print("Final CV: ", bacc*100)
def load(cls, weights_file, params_file, preprocessor_file): self = cls() self.p = IndexTransformer.load(preprocessor_file) self.model = load_model(weights_file, params_file) return self