def main(in_babi, in_babi_plus, in_result_folder):
babi_files = get_files_list(in_babi, 'task1-API-calls')
babi_plus_files = get_files_list(in_babi_plus, 'task1-API-calls')
if not os.path.exists(in_result_folder):
os.makedirs(in_result_folder)
for babi_file, babi_plus_file in zip(babi_files, babi_plus_files):
babi = read_task(babi_file)
babi_plus = read_task(babi_plus_file)
result = []
for babi_dialogue, babi_plus_dialogue in zip(babi, babi_plus):
for babi_turn, babi_plus_turn in zip(babi_dialogue[1],
babi_plus_dialogue[1]):
if babi_turn['agent'] == 'user':
result.append({
'babi': babi_turn['text'],
'babi_plus': babi_plus_turn['text']
})
with open(os.path.join(in_result_folder, os.path.basename(babi_file)),
'w') as result_out:
print >> result_out, ';'.join(['babi_plus', 'babi'])
for turn_pair in result:
print >> result_out, ';'.join(
[turn_pair['babi_plus'], turn_pair['babi']])
def main(in_babi, in_babi_plus, in_result_folder, in_output_format):
babi_files = get_files_list(in_babi, 'task1-API-calls')
babi_plus_files = get_files_list(in_babi_plus, 'task1-API-calls')
if not os.path.exists(in_result_folder):
os.makedirs(in_result_folder)
for babi_file, babi_plus_file in zip(babi_files, babi_plus_files):
babi = read_task(babi_file)
babi_plus = read_task(babi_plus_file)
result = []
for babi_dialogue, babi_plus_dialogue in zip(babi, babi_plus):
for babi_turn, babi_plus_turn in zip(babi_dialogue[1], babi_plus_dialogue[1]):
if babi_turn['agent'] == 'user' and babi_turn['text'].lower() != '<silence>':
result.append({
'babi': babi_turn['text'],
'babi_plus': babi_plus_turn['text']
})
if in_output_format == 'csv':
save_csv(result, os.path.join(in_result_folder, os.path.basename(babi_file)))
elif in_output_format == 'seq2seq':
result_folder = os.path.join(in_result_folder, os.path.basename(babi_file))
if not os.path.exists(result_folder):
os.makedirs(result_folder)
save_seq2seq(result, result_folder)
def collect_babi_slot_values(in_babi_root):
dataset_files = get_files_list(in_babi_root, 'task1-API-calls')
babi_files = [(filename, read_task(filename))
for filename in dataset_files]
full_babi = reduce(lambda x, y: x + y[1], babi_files, [])
slots_map = extract_slot_values(full_babi)
return reduce(lambda x, y: list(x) + list(y), slots_map.values(), [])
def plus_dataset(in_src_root, in_result_size):
dataset_files = get_files_list(in_src_root, 'task1-API-calls')
babi_files = [(filename, read_task(filename))
for filename in dataset_files]
full_babi = reduce(lambda x, y: x + y[1], babi_files, [])
slots_map = extract_slot_values(full_babi)
babi_plus = defaultdict(lambda: [])
result_size = in_result_size if in_result_size else len(babi_files)
for task_name, task in babi_files:
for dialogue_index, dialogue in zip(xrange(result_size), cycle(task)):
babi_plus[task_name].append(
augment_dialogue(dialogue, slots_map.values()))
return babi_plus
def collect_babi_slot_value_pps(in_babi_root, in_slot_values):
dataset_files = get_files_list(in_babi_root, 'task1-API-calls')
babi_files = [(filename, read_task(filename))
for filename in dataset_files]
full_babi = reduce(lambda x, y: x + y[1], babi_files, [])
return extract_slot_value_pps(full_babi, in_slot_values)
