def main(in_babi, in_babi_plus, in_result_folder): babi_files = get_files_list(in_babi, 'task1-API-calls') babi_plus_files = get_files_list(in_babi_plus, 'task1-API-calls') if not os.path.exists(in_result_folder): os.makedirs(in_result_folder) for babi_file, babi_plus_file in zip(babi_files, babi_plus_files): babi = read_task(babi_file) babi_plus = read_task(babi_plus_file) result = [] for babi_dialogue, babi_plus_dialogue in zip(babi, babi_plus): for babi_turn, babi_plus_turn in zip(babi_dialogue[1], babi_plus_dialogue[1]): if babi_turn['agent'] == 'user': result.append({ 'babi': babi_turn['text'], 'babi_plus': babi_plus_turn['text'] }) with open(os.path.join(in_result_folder, os.path.basename(babi_file)), 'w') as result_out: print >> result_out, ';'.join(['babi_plus', 'babi']) for turn_pair in result: print >> result_out, ';'.join( [turn_pair['babi_plus'], turn_pair['babi']])
def main(in_babi, in_babi_plus, in_result_folder, in_output_format): babi_files = get_files_list(in_babi, 'task1-API-calls') babi_plus_files = get_files_list(in_babi_plus, 'task1-API-calls') if not os.path.exists(in_result_folder): os.makedirs(in_result_folder) for babi_file, babi_plus_file in zip(babi_files, babi_plus_files): babi = read_task(babi_file) babi_plus = read_task(babi_plus_file) result = [] for babi_dialogue, babi_plus_dialogue in zip(babi, babi_plus): for babi_turn, babi_plus_turn in zip(babi_dialogue[1], babi_plus_dialogue[1]): if babi_turn['agent'] == 'user' and babi_turn['text'].lower() != '<silence>': result.append({ 'babi': babi_turn['text'], 'babi_plus': babi_plus_turn['text'] }) if in_output_format == 'csv': save_csv(result, os.path.join(in_result_folder, os.path.basename(babi_file))) elif in_output_format == 'seq2seq': result_folder = os.path.join(in_result_folder, os.path.basename(babi_file)) if not os.path.exists(result_folder): os.makedirs(result_folder) save_seq2seq(result, result_folder)
def collect_babi_slot_values(in_babi_root): dataset_files = get_files_list(in_babi_root, 'task1-API-calls') babi_files = [(filename, read_task(filename)) for filename in dataset_files] full_babi = reduce(lambda x, y: x + y[1], babi_files, []) slots_map = extract_slot_values(full_babi) return reduce(lambda x, y: list(x) + list(y), slots_map.values(), [])
def plus_dataset(in_src_root, in_result_size): dataset_files = get_files_list(in_src_root, 'task1-API-calls') babi_files = [(filename, read_task(filename)) for filename in dataset_files] full_babi = reduce(lambda x, y: x + y[1], babi_files, []) slots_map = extract_slot_values(full_babi) babi_plus = defaultdict(lambda: []) result_size = in_result_size if in_result_size else len(babi_files) for task_name, task in babi_files: for dialogue_index, dialogue in zip(xrange(result_size), cycle(task)): babi_plus[task_name].append( augment_dialogue(dialogue, slots_map.values())) return babi_plus
def collect_babi_slot_value_pps(in_babi_root, in_slot_values): dataset_files = get_files_list(in_babi_root, 'task1-API-calls') babi_files = [(filename, read_task(filename)) for filename in dataset_files] full_babi = reduce(lambda x, y: x + y[1], babi_files, []) return extract_slot_value_pps(full_babi, in_slot_values)