def test_batch_generator(batch_size=32): batch_size //= 2 index = 0 while index < test_num: sentences = [] batch_pos = test_pos_files[index:min(index + batch_size, test_num)] batch_neg = test_neg_files[index:min(index + batch_size, test_num)] index += batch_size for file_name in batch_pos: with codecs.open(os.path.join(TEST_ROOT, 'pos', file_name), 'r', 'utf8') as reader: text = reader.read().strip() sentences.append(get_word_list_eng(text)) for file_name in batch_neg: with codecs.open(os.path.join(TEST_ROOT, 'neg', file_name), 'r', 'utf8') as reader: text = reader.read().strip() sentences.append(get_word_list_eng(text)) yield wc_embd.get_batch_input(sentences)
print('Train: %d Validate: %d' % (train_num, val_num)) # Generate dictionaries for words and characters print('Get dictionaries....') wc_embd = WordCharEmbd( word_min_freq=5, char_min_freq=2, word_ignore_case=True, char_ignore_case=False, ) for file_name in train_pos_files: with codecs.open(os.path.join(TRAIN_ROOT, 'pos', file_name), 'r', 'utf8') as reader: text = reader.read().strip() wc_embd.update_dicts(get_word_list_eng(text)) for file_name in train_neg_files: with codecs.open(os.path.join(TRAIN_ROOT, 'neg', file_name), 'r', 'utf8') as reader: text = reader.read().strip() wc_embd.update_dicts(get_word_list_eng(text)) # Create model for classification print('Create model...') inputs, embd_layer = wc_embd.get_embedding_layer(word_embd_dim=150, char_embd_dim=30, char_hidden_dim=75, char_hidden_layer_type='lstm') lstm_layer = keras.layers.Bidirectional( keras.layers.LSTM(units=50), name='Bi-LSTM',