intents_files = [
basedir + train_intents_file, basedir + valid_intents_file,
basedir + test_intents_file
]
web_files = []
if add_web:
web_files = [
basedir + train_web,
basedir + valid_web,
basedir + test_web,
]
corpus, rels_train, rels_valid, rels_test, rels_intents, all_rels_web = prepare.run_with_train_valid_test_corpus_dmn(
basedir + train_file,
basedir + valid_file,
basedir + test_file,
intents_files=intents_files,
web_files=web_files)
for data_part in list(['train', 'valid', 'test']):
if data_part == 'train':
rels = rels_train
index = 0
elif data_part == 'valid':
rels = rels_valid
index = 1
else:
rels = rels_test
index = 2
print 'total relations in ', data_part, len(rels)
elif data_name == 'mantis_10':
train_file = 'data_train_easy.tsv'
valid_file = 'data_dev_easy.tsv'
test_file = 'data_test_easy.tsv'
elif data_name == 'mantis_50':
train_file = 'data_train.tsv'
valid_file = 'data_dev.tsv'
test_file = 'data_test.tsv'
elif data_name == 'L4':
train_file = 'train.tsv'
valid_file = 'dev.tsv'
test_file = 'test.tsv'
else:
raise ValueError('invalid data name!')
corpus, rels_train, rels_valid, rels_test = prepare.run_with_train_valid_test_corpus_dmn(
basedir + train_file, basedir + valid_file, basedir + test_file)
for data_part in list(['train', 'valid', 'test']):
if data_part == 'train':
rels = rels_train
elif data_part == 'valid':
rels = rels_valid
else:
rels = rels_test
print 'total relations in ', data_part, len(rels)
prepare.save_relation(cur_data_dir + 'relation_' + data_part + '.txt',
rels)
print 'filter queries with duplicated doc ids...'
prepare.check_filter_query_with_dup_doc(cur_data_dir + 'relation_' +
data_part + '.txt')
print 'total corpus ', len(corpus)
prepare.save_corpus_dmn(cur_data_dir + 'corpus.txt', corpus, '\t')