max_word_len = 13
word_dim = 300
adam = keras.optimizers.Adam(clipnorm=0.0001)
adamax = keras.optimizers.Adamax(clipnorm=0.0001)
'''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' ''''''
'''''' '''''' '''''' ''' create model '''
'''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' ''''''
dm = DataManager()
voc = Vocabulary()
dm.word_dim = word_dim
dm.word_len = max_word_len
voc.word2vec('data/w2v_model')
print("reading data...", end='')
dm.read_train_data('data/training_data/1_train.txt', 'train1')
dm.read_train_data('data/training_data/2_train.txt', 'train2')
dm.read_train_data('data/training_data/3_train.txt', 'train3')
dm.read_train_data('data/training_data/4_train.txt', 'train4')
dm.read_train_data('data/training_data/5_train.txt', 'train5')
dm.read_test_data('data/testing_data.csv', 'test_question', 'test_option')
print("\rreading data...finish")
print(dm.data['train1'][:3])
print(dm.data['train2'][:3])
print(dm.data['train3'][:3])
print(dm.data['train4'][:3])
print(dm.data['train5'][:3])
print("construct data...", end='')
dm.construct_data_LSTM('train1', voc, 'data/train1.npy')
dm.construct_data_LSTM('train2', voc, 'data/train2.npy')
word_dim = 300
adam = keras.optimizers.Adam(clipnorm=0.0001)
adamax = keras.optimizers.Adamax(clipnorm=0.0001)
'''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' ''''''
'''''' '''''' '''''' ''' create model '''
'''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' ''''''
dm = DataManager()
voc = Vocabulary()
dm.word_dim = word_dim
dm.word_len = max_word_len
voc.word2vec('data/w2v_model')
print("reading data...", end='')
dm.read_train_data('data/training_data/1_train.txt', 'train1')
dm.read_test_data('data/testing_data.csv', 'test_question', 'test_option')
print("\rreading data...finish")
print(dm.data['test_question'][:6])
print("construct data...")
dm.construct_data_seq2seq('train1', voc, 'data/train1.npy')
dm.construct_data_seq2seq('test_question', voc, 'data/test_question.npy')
dm.construct_data_seq2seq('test_option',
voc,
'data/test_option.npy',
multi_seq=True)
print("construct data...finish")
print('test_question_seq.shape: ' + str(dm.data['test_question'].shape))
print('test_option.shape: ' + str(dm.data['test_option'].shape))
'''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' ''''''