def test_local_textset_integration(self): local_set = LocalTextSet(self.texts, self.labels) assert local_set.is_local() assert not local_set.is_distributed() assert local_set.get_texts() == self.texts assert local_set.get_labels() == self.labels tokenized = ChainedPreprocessing([Tokenizer(), Normalizer()])(local_set) word_index = tokenized.generate_word_index_map(max_words_num=10) transformed = ChainedPreprocessing([WordIndexer(word_index), SequenceShaper(10), TextFeatureToSample()])(tokenized) assert transformed.is_local() word_index = transformed.get_word_index() assert len(word_index) == 10 assert word_index["my"] == 1 samples = transformed.get_samples() assert len(samples) == 3 for sample in samples: assert sample.feature.shape[0] == 10 vocab_file = create_tmp_path() + ".txt" transformed.save_word_index(vocab_file) local_set2 = LocalTextSet(self.texts, self.labels) local_set2.load_word_index(vocab_file) transformed2 = local_set2.tokenize().normalize().word2idx()\ .shape_sequence(10).generate_sample() samples2 = transformed2.get_samples() for s1, s2 in zip(samples, samples2): assert np.allclose(s1.feature.to_ndarray(), s2.feature.to_ndarray()) os.remove(vocab_file) model = TextClassifier(5, self.glove_path, word_index, 10) model.compile("adagrad", "sparse_categorical_crossentropy", ['accuracy']) tmp_log_dir = create_tmp_path() tmp_checkpoint_path = create_tmp_path() os.mkdir(tmp_checkpoint_path) model.set_tensorboard(tmp_log_dir, "textclassification") model.set_checkpoint(tmp_checkpoint_path) model.fit(transformed, batch_size=2, nb_epoch=2, validation_data=transformed) acc = model.evaluate(transformed, batch_size=2) res_set = model.predict(transformed, batch_per_thread=2) predicts = res_set.get_predicts() # Test for loaded model predict on TextSet tmp_path = create_tmp_path() + ".bigdl" model.save_model(tmp_path, over_write=True) loaded_model = TextClassifier.load_model(tmp_path) loaded_res_set = loaded_model.predict(transformed, batch_per_thread=2) loaded_predicts = loaded_res_set.get_predicts() assert len(predicts) == len(loaded_predicts) for i in range(0, len(predicts)): # (uri, prediction) assert not predicts[i][0] assert not loaded_predicts[i][0] # uri is not recorded and thus None assert len(predicts[i][1]) == 1 assert len(loaded_predicts[i][1]) == 1 assert predicts[i][1][0].shape == (5, ) assert np.allclose(predicts[i][1][0], loaded_predicts[i][1][0]) shutil.rmtree(tmp_log_dir) shutil.rmtree(tmp_checkpoint_path) os.remove(tmp_path)
def test_forward_backward(self): resource_path = os.path.join( os.path.split(__file__)[0], "../../resources") glove_path = os.path.join(resource_path, "glove.6B/glove.6B.50d.txt") model = TextClassifier(10, glove_path) model.summary() input_data = np.random.randint(20, size=(4, 500)) self.assert_forward_backward(model, input_data)
def test_distributed_textset_integration(self): texts_rdd = self.sc.parallelize(self.texts) labels_rdd = self.sc.parallelize(self.labels) distributed_set = DistributedTextSet(texts_rdd, labels_rdd) assert distributed_set.is_distributed() assert not distributed_set.is_local() assert distributed_set.get_texts().collect() == self.texts assert distributed_set.get_labels().collect() == self.labels sets = distributed_set.random_split([0.5, 0.5]) train_texts = sets[0].get_texts().collect() test_texts = sets[1].get_texts().collect() assert set(train_texts + test_texts) == set(self.texts) tokenized = Tokenizer()(distributed_set) transformed = tokenized.normalize().word2idx().shape_sequence( 5).generate_sample() word_index = transformed.get_word_index() assert len(word_index) == 14 samples = transformed.get_samples().collect() assert len(samples) == 3 for sample in samples: assert sample.feature.shape[0] == 5 vocab_file = create_tmp_path() + ".txt" transformed.save_word_index(vocab_file) distributed_set2 = DistributedTextSet(texts_rdd, labels_rdd) distributed_set2.load_word_index(vocab_file) transformed2 = distributed_set2.tokenize().normalize().word2idx()\ .shape_sequence(5).generate_sample() samples2 = transformed2.get_samples().collect() for s1, s2 in zip(samples, samples2): assert np.allclose(s1.feature.to_ndarray(), s2.feature.to_ndarray()) os.remove(vocab_file) model = TextClassifier(5, self.glove_path, word_index, 5, encoder="lstm") model.compile(SGD(), SparseCategoricalCrossEntropy()) model.fit(transformed, batch_size=2, nb_epoch=2) res_set = model.predict(transformed, batch_per_thread=2) predicts = res_set.get_predicts().collect() for predict in predicts: assert len(predict) == 1 assert predict[0].shape == (5, ) tmp_path = create_tmp_path() + ".bigdl" model.save_model(tmp_path, over_write=True) loaded_model = TextClassifier.load_model(tmp_path) loaded_res_set = loaded_model.predict(transformed, batch_per_thread=2) loaded_predicts = loaded_res_set.get_predicts().collect() assert len(loaded_predicts) == len(predicts) os.remove(tmp_path)
def test_forward_backward(self): model = TextClassifier(10, glove_path) model.summary() input_data = np.random.randint(20, size=(4, 500)) self.assert_forward_backward(model, input_data) model.set_evaluate_status() # Forward twice will get the same output output1 = model.forward(input_data) output2 = model.forward(input_data) assert np.allclose(output1, output2)
[float(options.training_split), 1 - float(options.training_split)]) if options.model: model = TextClassifier.load_model(options.model) else: token_length = int(options.token_length) if not (token_length == 50 or token_length == 100 or token_length == 200 or token_length == 300): raise ValueError( 'token_length for GloVe can only be 50, 100, 200, 300, but got ' + str(token_length)) embedding_file = options.embedding_path + "/glove.6B." + str( token_length) + "d.txt" word_index = transformed.get_word_index() model = TextClassifier(int(options.class_num), embedding_file, word_index, int(options.sequence_length), options.encoder, int(options.encoder_output_dim)) model.compile(optimizer=Adagrad(learningrate=float(options.learning_rate), learningrate_decay=0.001), loss="sparse_categorical_crossentropy", metrics=['accuracy']) app_name = 'textclassification-' + dt.datetime.now().strftime( "%Y%m%d-%H%M%S") model.set_tensorboard(options.log_dir, app_name) model.fit(train_set, batch_size=int(options.batch_size), nb_epoch=int(options.nb_epoch), validation_data=val_set) predict_set = model.predict(val_set, batch_per_thread=int(options.partition_num))
def test_save_load(self): model = TextClassifier(20, 200) input_data = np.random.random([2, 500, 200]) self.assert_zoo_model_save_load(model, input_data)
def test_forward_backward(self): model = TextClassifier(10, 30, 100) model.summary() input_data = np.random.random([3, 100, 30]) self.assert_forward_backward(model, input_data)
tokens_label[0], "##", sequence_len), tokens_label[1])) vector_rdd = padded_tokens_rdd.map(lambda tokens_label: ([ to_vec(w, filtered_word2vec_broadcast.value, token_length) for w in tokens_label[0] ], tokens_label[1])) sample_rdd = vector_rdd.map(lambda vectors_label: to_sample( vectors_label[0], vectors_label[1], token_length)) train_rdd, val_rdd = sample_rdd.randomSplit( [training_split, 1 - training_split]) if options.model: model = TextClassifier.load_model(options.model) else: model = TextClassifier(CLASS_NUM, token_length, sequence_len, options.encoder, int(options.encoder_output_dim)) optimizer = Optimizer(model=model, training_rdd=train_rdd, criterion=ClassNLLCriterion(logProbAsInput=False), end_trigger=MaxEpoch(int(options.nb_epoch)), batch_size=batch_size, optim_method=Adagrad(learningrate=float( options.learning_rate), learningrate_decay=0.001)) optimizer.set_validation(batch_size=batch_size, val_rdd=val_rdd, trigger=EveryEpoch(), val_method=[Top1Accuracy()])
train_rdd, val_rdd = sample_rdd.randomSplit( [training_split, 1 - training_split]) if options.model: model = TextClassifier.load_model(options.model) else: if not (token_length == 50 or token_length == 100 or token_length == 200 or token_length == 300): raise ValueError( 'token_length for GloVe can only be 50, 100, 200, 300, but got ' + str(token_length)) embedding_file = data_path + "/glove.6B/glove.6B." + str( token_length) + "d.txt" word_index = {w: i_f[0] for w, i_f in word_meta.items()} model = TextClassifier(class_num, embedding_file, word_index, sequence_len, options.encoder, int(options.encoder_output_dim)) optimizer = Optimizer(model=model, training_rdd=train_rdd, criterion=SparseCategoricalCrossEntropy(), end_trigger=MaxEpoch(int(options.nb_epoch)), batch_size=batch_size, optim_method=Adagrad(learningrate=float( options.learning_rate), learningrate_decay=0.001)) optimizer.set_validation(batch_size=batch_size, val_rdd=val_rdd, trigger=EveryEpoch(), val_method=[Accuracy()])
def test_save_load(self): model = TextClassifier(20, glove_path, sequence_length=100) input_data = np.random.randint(20, size=(3, 100)) self.assert_zoo_model_save_load(model, input_data)
def test_forward_backward(self): model = TextClassifier(10, glove_path) model.summary() input_data = np.random.randint(20, size=(4, 500)) self.assert_forward_backward(model, input_data)