sentences, sentence_length = pad_sentences( conversations, max_sentence_length=max_sent_len, max_conversation_length=max_conv_len) print('Saving preprocessed data at', split_data_dir) to_pickle(conversation_length, split_data_dir.joinpath('conversation_length.pkl')) to_pickle(sentences, split_data_dir.joinpath('sentences.pkl')) to_pickle(sentence_length, split_data_dir.joinpath('sentence_length.pkl')) to_pickle(emotions, split_data_dir.joinpath('labels.pkl')) if split_type == 'train': print('Save Vocabulary...') vocab = Vocab(tokenizer) vocab.add_dataframe(conversations) assert(GLOVE_DIR != "") vocab.update(GLOVE_DIR, max_size=max_vocab_size, min_freq=min_freq) print('Vocabulary size: ', len(vocab)) vocab.pickle(dailydialog_dir.joinpath('word2id.pkl'), dailydialog_dir.joinpath('id2word.pkl'), dailydialog_dir.joinpath('word_emb.pkl')) print('Done!')
('test', test)]: print(f'Processing {split_type} dataset...') split_data_dir = datasets_dir.joinpath(split_type) split_data_dir.mkdir(exist_ok=True) conversation_length = [ min(len(conv), max_conv_len) for conv in conversations ] sentences, sentence_length = pad_sentences( conversations, max_sentence_length=max_sent_len, max_conversation_length=max_conv_len) print('Saving preprocessed data at', split_data_dir) to_pickle(conversation_length, split_data_dir.joinpath('conversation_length.pkl')) to_pickle(sentences, split_data_dir.joinpath('sentences.pkl')) to_pickle(sentence_length, split_data_dir.joinpath('sentence_length.pkl')) if split_type != 'test': print('Save Vocabulary...') vocab.add_dataframe(conversations) vocab.update(max_size=max_vocab_size, min_freq=min_freq) print('Vocabulary size: ', len(vocab)) vocab.pickle(datasets_dir.joinpath('word2id.pkl'), datasets_dir.joinpath('id2word.pkl')) print('Done!')