Python DataManager.construct_data_seq2seq Examples

Example #1

File: test_simple_avg.py Project: CheNgH826/Conversation_MLfinal

max_word_len = 14
word_dim_list = [50, 100, 150, 200, 250, 300, 350, 400]
test = np.zeros((5060, 6))

for word_dim in word_dim_list:
    print('word dim=', word_dim)
    dm = DataManager()
    voc = Vocabulary()
    dm.word_dim = word_dim
    dm.word_len = max_word_len

    voc.word2vec('data/w2v_model/w2v_model_{}'.format(word_dim))
    print("reading data...", end='')
    dm.read_test_data('data/testing_data.csv', 'test_question', 'test_option')
    print("\rreading data...finish")

    print("construct data...")
    dm.construct_data_seq2seq('test_question', voc, 'data/test_question.npy')
    dm.construct_data_seq2seq('test_option',
                              voc,
                              'data/test_option.npy',
                              multi_seq=True)
    print("construct data...finish")
    print('test_question_seq.shape: ' + str(dm.data['test_question'].shape))
    print('test_option.shape: ' + str(dm.data['test_option'].shape))

    test = dm.output(dm.data['test_question'])
    test_y = np.argmax(test, axis=1)
    dm.write(test_y, 'ans_{}.csv'.format(word_dim))

Example #2

print("reading data...", end='')
dm.read_train_data('data/training_data/1_train.txt', 'train1')
dm.read_train_data('data/training_data/2_train.txt', 'train2')
dm.read_train_data('data/training_data/3_train.txt', 'train3')
dm.read_train_data('data/training_data/4_train.txt', 'train4')
dm.read_train_data('data/training_data/5_train.txt', 'train5')
#dm.read_test_data('data/testing_data.csv','test_question','test_option')
print("\rreading data...finish")
print(dm.data['train1'][:3])
print(dm.data['train2'][:3])
print(dm.data['train3'][:3])
print(dm.data['train4'][:3])
print(dm.data['train5'][:3])

print("construct data...")
dm.construct_data_seq2seq('train1', voc, 'data/train1_seq.npy')
dm.construct_data_seq2seq('train2', voc, 'data/train2_seq.npy')
dm.construct_data_seq2seq('train3', voc, 'data/train3_seq.npy')
dm.construct_data_seq2seq('train4', voc, 'data/train4_seq.npy')
dm.construct_data_seq2seq('train5', voc, 'data/train5_seq.npy')
#dm.construct_data_seq2seq('test_question',voc,'data/test_question_seq.npy')
#dm.construct_data_seq2seq('test_option',voc,'data/test_option_seq.npy',multi_seq=True)
print("construct data...finish")

model = dm.construct_seq2seq_train(1024)
'''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' ''''''
'''''' '''''' '''''' '''       compile model                            '''
'''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' ''''''
model.compile(optimizer=adam, loss='cosine_proximity', metrics=['accuracy'])
'''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' ''''''
'''''' '''''' '''''' '''       setting checkpoint                       '''