def input_files(path, filepath=None, relex=False): """ Read the corpus, write train and dev files. :param path: directory with the WebNLG benchmark :param filepath: path to the prediction file with sentences (for relexicalisation) :param relex: boolean; do relexicalisation or not :return: """ parts = ['train', 'dev', 'test'] options = ['all-delex', 'all-notdelex'] # generate files with/without delexicalisation for part in parts: for option in options: files = select_files(path + part, size=(1, 8)) b = Benchmark() b.fill_benchmark(files) if option == 'all-delex': rplc_list = create_source_target(b, option, part, delex=True) print('Total of {} files processed in {} with {} mode'.format( len(files), part, option)) elif option == 'all-notdelex': rplc_list = create_source_target(b, option, part, delex=False) print('Total of {} files processed in {} with {} mode'.format( len(files), part, option)) if relex and part == 'dev' and option == 'all-delex': relexicalise(filepath, rplc_list) print('Files necessary for training/evaluating are written on disc.')
def input_files(path_org, path_pre, filepath, part): files = select_files(path_org + part, size=(1, 8)) b = Benchmark() b.fill_benchmark(files) rplc_list = create_source_target(b, 'all-delex', part, delex=True) relexicalise(filepath, rplc_list, path_pre, part) print('Files necessary for training/evaluating are written on disc.')
def read_rdf_file(file, mode='train'): nodes = [] # [batch, node_num,] in_neigh_indices = [] # [batch, node_num, neighbor_num,] in_neigh_edges = [] out_neigh_indices = [] # [batch, node_num, neighbor_num,] out_neigh_edges = [] sentences = [] # [batch, sent_length,] ids = [] max_in_neigh = 0 max_out_neigh = 0 max_node = 0 max_sent = 0 rplc_list_dev_delex = None options = ['all-notdelex'] # generate files with/without delexicalisation for option in options: b = Benchmark() b.fill_benchmark(file) if mode == 'test': all_instances, max_node, max_in_neigh, max_out_neigh, max_sent = create_test_data( b, option, delex=False) elif option == 'all-delex': all_instances, max_node, max_in_neigh, max_out_neigh, max_sent = create_source_target( b, option, file.split('.')[0], delex=True) else: all_instances, max_node, max_in_neigh, max_out_neigh, max_sent = create_source_target( b, option, file.split('.')[0], delex=False) return all_instances, max_node, max_in_neigh, max_out_neigh, max_sent
def read_participant(output_file, teamname): # read participant's outputs output = [] with open(output_file, 'r') as f: output += [unidecode(line.strip()) for line in f] b = Benchmark() b.fill_benchmark([(path, goldfile)]) # per size for size in range(1, 8): # print(size) # print('# of instances', b.entry_count(size=str(size))) entry_ids = [] # look up id of a line in the gold benchmark, extract its size for entry in b.entries: if int(entry.size) == size: entry_ids += [int(entry.id[2:])] # entry id -- 'Id1' output_reduced = [output[i - 1] for i in sorted(entry_ids)] write_to_file(output_reduced, str(size) + 'size', teamname) # per category for category in categories: # print(category) # print('# of instances', b.entry_count(cat=category)) entry_ids = [] # look up id of a line in the gold benchmark, extract its category for entry in b.entries: if entry.category == category: entry_ids += [int(entry.id[2:])] # entry id -- 'Id1' output_reduced = [output[i - 1] for i in sorted(entry_ids)] write_to_file(output_reduced, category, teamname) # old categories entry_ids = [] for category in old_categories: # print('# of instances', b.entry_count(cat=category)) # look up id of a line in the gold benchmark, extract its category for entry in b.entries: if entry.category == category: entry_ids += [int(entry.id[2:])] # entry id -- 'Id1' output_reduced = [output[i - 1] for i in sorted(entry_ids)] write_to_file(output_reduced, 'old-cat', teamname) # new categories entry_ids = [] for category in new_categories: # print('# of instances', b.entry_count(cat=category)) # look up id of a line in the gold benchmark, extract its category for entry in b.entries: if entry.category == category: entry_ids += [int(entry.id[2:])] # entry id -- 'Id1' output_reduced = [output[i - 1] for i in sorted(entry_ids)] write_to_file(output_reduced, 'new-cat', teamname) # create all-category files write_to_file(output, 'all-cat', teamname) print('Files creating finished for: ', teamname)
def input_files(path, filepath=None, relex=False): """ Read the corpus, write train and dev files. :param path: directory with the WebNLG benchmark :param filepath: path to the prediction file with sentences (for relexicalisation) :param relex: boolean; do relexicalisation or not :return: """ part = 'dev' options = ['all-delex', 'all-notdelex'] # generate files with/without delexicalisation #files is list of list of tuple for all files in each triple folder for option in options: files = select_files(path, size=(1, 8)) #take general list of files # print(files) i = 0 for file1 in files: #take each list of tuples i += 1 b = Benchmark() #print(file1) b.fill_benchmark(file1) #read all xml files in triple1 for example print('Its triple size' + str(i)) #create new folder for each triple size folder = 'dev_triple' + str(i) if not os.path.exists(folder): os.makedirs(folder) if option == 'all-delex': rplc_list = create_source_target(b, i, option, folder, part, delex=True) print('Total of {} files processed in {} with {} mode'.format( len(file1), part, option)) elif option == 'all-notdelex': rplc_list = create_source_target(b, i, option, folder, part, delex=False) print('Total of {} files processed in {} with {} mode'.format( len(file1), part, option)) # take each pred file for current triple size if relex and part == 'dev' and option == 'all-delex': sizeT = i filepath = folder + '/' + 'baseline_pred_triple' + str( sizeT) + '.txt' print('Path to the file is', filepath) relexicalise(filepath, folder, sizeT, rplc_list) if ONLINE: with open('delex_dict_dbpd.json', 'w') as f: json.dump(delex_dict_dbpd, f) print('Files necessary for evaluating are written on disc.')
def input_files(path, filepath=None, relex=False): """ Read the corpus, write files for chosen DBpedia category. :param path: directory with the WebNLG benchmark :param filepath: path to the prediction file with sentences (for relexicalisation) :param relex: boolean; do relexicalisation or not :return: """ part = 'dev' options = ['all-delex', 'all-notdelex'] # generate files with/without delexicalisation categories = ['Astronaut', 'University', 'Food', 'SportsTeam', 'City'] for category in categories: #create new folder for current category folder = 'dev_' + category if not os.path.exists(folder): os.makedirs(folder) for option in options: files = select_files(path, category, size=(1, 8)) b = Benchmark() b.fill_benchmark(files) if option == 'all-delex': rplc_list = create_source_target(b, option, folder, part, delex=True) print('Total of {} files processed in {} with {} mode'.format( len(files), category, option)) elif option == 'all-notdelex': rplc_list = create_source_target(b, option, folder, part, delex=False) print('Total of {} files processed in {} with {} mode'.format( len(files), category, option)) # take each pred file for current triple size if relex and part == 'dev' and option == 'all-delex': #filepath:path to find the output of seq2seq prediction file filepath = folder + '/baseline_pred.txt' print('Path to the file is', filepath) relexicalise(filepath, folder, rplc_list) #files is list of list of tuple for all files in each triple folder if ONLINE: with open('delex_dict_dbpd.json', 'w') as f: json.dump(delex_dict_dbpd, f) print('Files necessary for evaluating are written on disc.')
r = r.replace('"', '').replace('_', ' ') cur_triples.append((h, r, t)) tgt = process_tgt_test(entry.lexs) src = process_src(cur_triples) wf_src.write(src + '\n') wf_tgt.write(json.dumps(tgt) + '\n') wf_tgt.close() wf_src.close() return eids outdir = 'data/webnlg_tag' b = Benchmark() files = [('webnlg_challenge_2017/test', 'testdata_unseen_with_lex.xml')] b.fill_benchmark(files) pair_valid_src = os.path.join(outdir, "unseen.source") pair_valid_tgt = os.path.join(outdir, "unseen.target") eids = convert_dataset_test(pair_valid_src, pair_valid_tgt, b) a = Benchmark() files = [('webnlg_challenge_2017/test', 'testdata_with_lex.xml')] a.fill_benchmark(files) pair_valid_src = os.path.join(outdir, "seen.source") pair_valid_tgt = os.path.join(outdir, "seen.target") eids = convert_dataset_test_1(pair_valid_src, pair_valid_tgt, a, eids) # files =[('webnlg_challenge_2017/test', 'testdata_with_lex.xml')] # pair_valid_src = os.path.join(outdir, "test.source") # pair_valid_tgt = os.path.join(outdir, "test.target") # eids = convert_dataset_test(pair_valid_src, pair_valid_tgt, a)
def generate_files(): # generate files per category for cat in categories: b = Benchmark() b.fill_benchmark([(path, goldfile)]) # print(cat + ': ' + str(b.entry_count(cat=cat))) b_reduced = b.filter([], [cat]) # print('reduced', b_reduced.entry_count(cat=cat)) # metric files generation; we use three references bleu_ref_files_gen(b_reduced, cat) # meteor_ref_files_gen(b_reduced, cat) meteor_3ref_files_gen(b_reduced, cat) # ter_ref_files_gen(b_reduced, cat) ter_ref_files_gen(b_reduced, cat, True) # ter_3ref_space_files_gen(b_reduced, cat) # generate files per size for size in range(1, 8): b = Benchmark() b.fill_benchmark([(path, goldfile)]) # print(str(size) + ': ' + str(b.entry_count(size=str(size)))) b_reduced = b.filter([size], []) # print('reduced', b_reduced.entry_count(size=str(size))) bleu_ref_files_gen(b_reduced, str(size) + 'size') # meteor_ref_files_gen(b_reduced, str(size) + 'size') meteor_3ref_files_gen(b_reduced, str(size) + 'size') # ter_ref_files_gen(b_reduced, str(size) + 'size') ter_ref_files_gen(b_reduced, str(size) + 'size', True) # ter_3ref_space_files_gen(b_reduced, str(size) + 'size') # generate files per type: old, new, all categories b = Benchmark() b.fill_benchmark([(path, goldfile)]) print('Gold count', b.entry_count()) # metric files generation for all cats bleu_ref_files_gen(b, 'all-cat') ter_3ref_space_files_gen( b, 'all-cat') # need this format for significance testing # meteor_ref_files_gen(b, 'all-cat') meteor_3ref_files_gen(b, 'all-cat') # ter_ref_files_gen(b, 'all-cat') ter_ref_files_gen(b, 'all-cat', True) b_reduced = b.filter([], new_categories) print('reduced (new)', b_reduced.entry_count()) # metric files generation for new cats bleu_ref_files_gen(b_reduced, 'new-cat') ter_3ref_space_files_gen( b_reduced, 'new-cat') # need this format for significance testing # meteor_ref_files_gen(b_reduced, 'new-cat') meteor_3ref_files_gen(b_reduced, 'new-cat') # ter_ref_files_gen(b_reduced, 'new-cat') ter_ref_files_gen(b_reduced, 'new-cat', True) bk = Benchmark() bk.fill_benchmark([(path, goldfile)]) bk_reduced = bk.filter([], old_categories) print('reduced (old)', bk_reduced.entry_count()) # metric files generation for old cats bleu_ref_files_gen(bk_reduced, 'old-cat') ter_3ref_space_files_gen( bk_reduced, 'old-cat') # need this format for significance testing # meteor_ref_files_gen(bk_reduced, 'old-cat') meteor_3ref_files_gen(bk_reduced, 'old-cat') # ter_ref_files_gen(bk_reduced, 'old-cat') ter_ref_files_gen(bk_reduced, 'old-cat', True)
def input_files(path, filepath=None, relex=False, parts=['train', 'dev'], doCategory=[], negraph=True, lowercased=True, fileid=None): """ Read the corpus, write train and dev files. :param path: directory with the WebNLG benchmark :param filepath: path to the prediction file with sentences (for relexicalisation) :param relex: boolean; do relexicalisation or not :param parts: partition to process :param negraph: whether to add edges for multi-word entitites :param lowercased: whether to do all lowercased for the notdelex version of the files :return: """ rplc_list_dev_delex = None options = ['all-delex', 'all-notdelex'] # generate files with/without delexicalisation for part in parts: for option in options: if part.startswith('test'): files = select_files(path + part, size=0) else: files = select_files(path + part, size=(1, 8)) if doCategory == UNSEEN_CATEGORIES: files = [files[1]] b = Benchmark() b.fill_benchmark(files) if option == 'all-delex': rplc_list = create_source_target(b, option, part, delex=True, relex=relex, doCategory=doCategory, negraph=negraph, lowercased=False) print('Total of {} files processed in {} with {} mode'.format( len(files), part, option)) elif option == 'all-notdelex': rplc_list = create_source_target(b, option, part, delex=False, relex=relex, doCategory=doCategory, negraph=negraph, lowercased=lowercased) print('Total of {} files processed in {} with {} mode'.format( len(files), part, option)) if (part == 'dev' or part.startswith('test')) and option == 'all-delex': rplc_list_dev_delex = rplc_list if relex and rplc_list_dev_delex: relexicalise(filepath, rplc_list_dev_delex, fileid, part, lowercased=lowercased) print('Files necessary for training/evaluating are written on disc.')
def input_files(path, filepath=None, relex=False, parts=['train', 'dev'], doCategory=[], options=None, negraph=True, lowercased=True, fileid=None): """ Read the corpus, write train and dev files. :param path: directory with the WebNLG benchmark :param filepath: path to the prediction file with sentences (for relexicalisation) :param relex: boolean; do relexicalisation or not :param parts: partition to process :param negraph: whether to add edges for multi-word entitites :param lowercased: whether to do all lowercased for the notdelex version of the files :return: """ rplc_list_relex = None # options = ['all-delex', 'all-notdelex'] # generate files with/without delexicalisation if options is None: options = ['all-delex', 'all-notdelex' ] # generate files with/without delexicalisation else: assert isinstance(options, list) assert len(set(options) - {'all-notdelex', 'all-delex'}) == 0 for part in parts: for option in options: files = select_files(path + part, size=(1, 8)) b = Benchmark() b.fill_benchmark(files) if option == 'all-delex': rplc_list = create_source_target(b, option, part, delex=True, relex=relex, doCategory=doCategory, negraph=negraph, lowercased=lowercased, is_train=(part == "train")) print('Total of {} instances processed in {} with {} mode'. format(len(rplc_list), part, option)) elif option == 'all-notdelex': rplc_list = create_source_target(b, option, part, delex=False, relex=relex, doCategory=doCategory, negraph=negraph, lowercased=lowercased, is_train=(part == "train")) print('Total of {} instances processed in {} with {} mode'. format(len(rplc_list), part, option)) if option == 'all-delex': rplc_list_relex = rplc_list if relex and rplc_list_relex: relexicalise(filepath, rplc_list_relex, fileid, part, lowercased=lowercased, doCategory=doCategory) print('Files necessary for training/evaluating are written on disc.')