def prepro(tokenizer, d, max_len=150, max_sent_len=60): """ make sure data is not empty""" source_sents, extracts = d tokenized_sents = [ tokenizer.tokenize(source_sent.lower()) for source_sent in source_sents ] tokenized_sents = tokenized_sents[:max_sent_len] tokenized_sents = [['[CLS]'] + tokenized_sent[:max_len - 1] for tokenized_sent in tokenized_sents] tokenized_sents = [ tokenizer.convert_tokens_to_ids(tokenized_sent) for tokenized_sent in tokenized_sents ] word_num = [ len(tokenized_sent) for tokenized_sent in tokenized_sents ] tokenized_sents = [ tokenizer.convert_tokens_to_ids(tokenized_sent) for tokenized_sent in tokenized_sents ] abs_sents = tokenize(None, extracts) art_sents = tokenize(None, source_sents) return (art_sents, tokenized_sents, word_num), abs_sents
def prepro(tokenizer, d, max_len=512): """ make sure data is not empty""" source_sents, extracts = d tokenized_sents = [ tokenizer.tokenize(source_sent.lower()) for source_sent in source_sents ] tokenized_sents = [ tokenized_sent + ['[SEP]'] for tokenized_sent in tokenized_sents ] tokenized_sents[0] = ['[CLS]'] + tokenized_sents[0] word_num = [ len(tokenized_sent) for tokenized_sent in tokenized_sents ] truncated_word_num = [] total_count = 0 for num in word_num: if total_count + num < max_len: truncated_word_num.append(num) else: truncated_word_num.append(512 - total_count) break total_count += num tokenized_sents = list(concat(tokenized_sents))[:max_len] tokenized_sents = tokenizer.convert_tokens_to_ids(tokenized_sents) abs_sents = tokenize(None, extracts) art_sents = tokenize(None, source_sents) return (art_sents, tokenized_sents, truncated_word_num), abs_sents
def coll(batch): art_batch, abs_batch, query_batch = unzip(batch) query_batch_list = list(query_batch) art_sents = list(filter(bool, map(tokenize(None), art_batch))) abs_sents = list(filter(bool, map(tokenize(None), abs_batch))) queries = list( filter( bool, map(tokenize(None), [[query] for query in query_batch_list]))) return art_sents, abs_sents, queries
def coll(batch): def is_good_data(d): """ make sure data is not empty""" source_sents, extracts = d return source_sents and extracts art_batch, abs_batch = unzip(batch) art_batch, abs_batch = list( zip(*list(filter(is_good_data, zip(art_batch, abs_batch))))) art_sents = list(filter(bool, map(tokenize(None), art_batch))) abs_sents = list(filter(bool, map(tokenize(None), abs_batch))) return art_sents, abs_sents
def prepro(tokenizer, d, max_len=1024, stride=256): """ make sure data is not empty""" source_sents, extracts = d tokenized_sents = [ tokenizer.tokenize(source_sent.lower()) for source_sent in source_sents ] tokenized_sents = [['[CLS]'] + tokenized_sent for tokenized_sent in tokenized_sents] tokenized_sents = [ tokenizer.convert_tokens_to_ids(tokenized_sent) for tokenized_sent in tokenized_sents ] word_num = [ len(tokenized_sent) for tokenized_sent in tokenized_sents ] truncated_word_num = [] total_count = 0 for num in word_num: if total_count + num < max_len: truncated_word_num.append(num) else: truncated_word_num.append(max_len - total_count) break total_count += num tokenized_sents = list(concat(tokenized_sents))[:max_len] tokenized_sents_lists = [tokenized_sents[:BERT_MAX_LEN]] length = len(tokenized_sents) - BERT_MAX_LEN i = 1 while length > 0: tokenized_sents_lists.append( tokenized_sents[(i * BERT_MAX_LEN - stride):((i + 1) * BERT_MAX_LEN - stride)]) i += 1 length -= (BERT_MAX_LEN - stride) abs_sents = tokenize(None, extracts) art_sents = tokenize(None, source_sents) return (art_sents, tokenized_sents_lists, truncated_word_num), abs_sents
def main(article_path, model_dir, batch_size, beam_size, diverse, max_len, cuda): with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = identity else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda) extractor = RLExtractor(model_dir, cuda=cuda) with open(article_path) as f: raw_article_batch = f.readlines() tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: print(raw_art_sents) ext = extractor(raw_art_sents)[:-1] # exclude EOE if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] else: ext = [i.item() for i in ext] ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] if beam_size > 1: all_beams = abstractor(ext_arts, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) else: dec_outs = abstractor(ext_arts) # assert i == batch_size*i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j + n]] print(decoded_sents)
def coll(batch): art_batch, abs_batch, i_batch = unzip(batch) art_sents = list(filter(bool, map(tokenize(None), art_batch))) abs_sents = list(filter(bool, map(tokenize(None), abs_batch))) return art_sents, abs_sents, list(i_batch)
def coll(batch): art_batch, topics, abs_batch = unzip(batch) #art_batch, topics, abs_batch, topic_label = unzip(batch) art_sents = list(filter(bool, map(tokenize(None), art_batch))) abs_sents = list(filter(bool, map(tokenize(None), abs_batch))) return art_sents, topics, abs_sents
def coll(batch): split_token = '<split>' pad = 0 art_batch, abs_batch, all_clusters = unzip(batch) art_sents = [] abs_sents = [] def is_good_data(d): """ make sure data is not empty""" source_sents, extracts = d return source_sents and extracts art_batch, abs_batch = list( zip(*list(filter(is_good_data, zip(art_batch, abs_batch))))) art_sents = list(filter(bool, map(tokenize(None), art_batch))) abs_sents = list(filter(bool, map(tokenize(None), abs_batch))) inputs = [] # merge cluster for art_sent, clusters in zip(art_sents, all_clusters): cluster_words = [] cluster_wpos = [] cluster_spos = [] for cluster in clusters: scluster_word = [] scluster_wpos = [] scluster_spos = [] for mention in cluster: if len(mention['text'].strip().split(' ')) == len( list( range(mention['position'][3] + 1, mention['position'][4] + 1))): scluster_word += mention['text'].lower().strip().split( ' ') scluster_wpos += list( range(mention['position'][3] + 1, mention['position'][4] + 1)) scluster_spos += [ mention['position'][0] + 1 for _ in range( len(mention['text'].strip().split(' '))) ] scluster_word.append(split_token) scluster_wpos.append(pad) scluster_spos.append(pad) else: sent_num = mention['position'][0] word_start = mention['position'][3] word_end = mention['position'][4] # if word_end > 99: # word_end = 99 if sent_num > len(art_sent) - 1: print('bad cluster') continue scluster_word += art_sent[sent_num][ word_start:word_end] scluster_wpos += list(range(word_start, word_end)) scluster_spos += [ mention['position'][0] + 1 for _ in range(word_start + 1, word_end + 1) ] scluster_word.append(split_token) scluster_wpos.append(pad) scluster_spos.append(pad) if scluster_word != []: scluster_word.pop() scluster_wpos.pop() scluster_spos.pop() cluster_words.append(scluster_word) cluster_wpos.append(scluster_wpos) cluster_spos.append(scluster_spos) if len(scluster_word) != len(scluster_wpos): print(scluster_word) print(scluster_wpos) print('cluster:', cluster) if len(scluster_word) != len(scluster_spos): print(scluster_word) print(scluster_spos) print('cluster:', cluster) assert len(scluster_word) == len(scluster_spos) and len( scluster_spos) == len(scluster_wpos) new_clusters = (cluster_words, cluster_wpos, cluster_spos) inputs.append((art_sent, new_clusters)) assert len(inputs) == len(abs_sents) return inputs, abs_sents
def decode(save_path, model_dir, split, batch_size, beam_size, diverse, max_len, cuda, bart=False, clip=-1, tri_block=False): start = time() # setup model with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = identity else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda) extractor = RLExtractor(model_dir, cuda=cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll) # prepare save paths and logs os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['abstractor'] = meta['net_args']['abstractor'] dec_log['extractor'] = meta['net_args']['extractor'] dec_log['rl'] = True dec_log['split'] = split dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): # raw_article_batch tokenized_article_batch = map(tokenize(None), [r[0] for r in raw_article_batch]) tokenized_abs_batch = map(tokenize(None), [r[1] for r in raw_article_batch]) ext_arts = [] ext_inds = [] for raw_art_sents, raw_abs_sents in zip(tokenized_article_batch, tokenized_abs_batch): ext, raw_art_sents = extractor(raw_art_sents, raw_abs_sents=raw_abs_sents) # print(raw_art_sen/ts) ext = ext[:-1] # exclude EOE # print(ext) if tri_block: _pred = [] _ids = [] for j in ext: if (j >= len(raw_art_sents)): continue candidate = " ".join(raw_art_sents[j]).strip() if (not _block_tri(candidate, _pred)): _pred.append(candidate) _ids.append(j) else: continue if (len(_pred) == 3): break ext = _ids # print(_pred) if clip > 0 and len( ext) > clip: #ADDED FOR CLIPPING, CHANGE BACK # print("hi", clip) ext = ext[0:clip] if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] else: ext = [i.item() for i in ext] # print(ext) ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] if bart: # print("hi") dec_outs = get_bart_summaries(ext_arts, tokenizer, bart_model, beam_size=beam_size) else: if beam_size > 1: all_beams = abstractor(ext_arts, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) else: dec_outs = abstractor(ext_arts) # print(dec_outs, i, i_debug) assert i == batch_size * i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j + n]] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') print()
def decode(save_path, abs_dir, split, batch_size, max_len, cuda, min_len): start = time() # setup model if abs_dir is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization raise Exception('abs directory none!') else: #abstractor = Abstractor(abs_dir, max_len, cuda) abstractor = BeamAbstractor(abs_dir, max_len, cuda, min_len, reverse=args.reverse) bert = abstractor._bert if bert: tokenizer = abstractor._tokenizer if bert: import logging logging.basicConfig(level=logging.ERROR) # if args.docgraph or args.paragraph: # abstractor = BeamAbstractorGAT(abs_dir, max_len, cuda, min_len, reverse=args.reverse) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = AbsDecodeDataset(split) n_data = len(dataset) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll) save_path = os.path.join(save_path, split) os.makedirs(save_path) # prepare save paths and logs dec_log = {} dec_log['abstractor'] = (None if abs_dir is None else json.load( open(join(abs_dir, 'meta.json')))) dec_log['rl'] = False dec_log['split'] = split dec_log['beam'] = 5 # greedy decoding only beam_size = 5 with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) os.makedirs(join(save_path, 'output')) # Decoding i = 0 length = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): if bert: tokenized_article_batch = map( tokenize_keepcase(args.max_input), raw_article_batch) else: tokenized_article_batch = map(tokenize(args.max_input), raw_article_batch) ext_arts = [] ext_inds = [] beam_inds = [] pre_abs = list(tokenized_article_batch) pre_abs = [article[0] for article in pre_abs] for j in range(len(pre_abs)): beam_inds += [(len(beam_inds), 1)] all_beams = abstractor(pre_abs, beam_size, diverse=1.0) dec_outs = rerank_mp(all_beams, beam_inds) for dec_out in dec_outs: if bert: text = ''.join(' '.join(dec_out).split(' ')) dec_out = bytearray([ tokenizer.byte_decoder[c] for c in text ]).decode('utf-8', errors=tokenizer.errors) dec_out = [dec_out] dec_out = sent_tokenize(' '.join(dec_out)) ext = [sent.split(' ') for sent in dec_out] ext_inds += [(len(ext_arts), len(ext))] ext_arts += ext dec_outs = ext_arts assert i == batch_size * i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j + n]] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') length += len(decoded_sents) print('average summary length:', length / i)
def prepro_subgraph_nobert(batch, max_sent_len=100, max_sent=60, node_max_len=30, key='InSalientSent', adj_type='edge_as_node'): source_sents, nodes, edges, subgraphs, paras, extracts = batch tokenized_sents = tokenize(max_sent_len, source_sents)[:max_sent] tokenized_sents_2 = tokenize(None, source_sents)[:max_sent] tokenized_article = list(concat(tokenized_sents_2)) cleaned_extracts = list( filter(lambda e: e < len(tokenized_sents), extracts)) max_len = len(tokenized_article) # tokenized_sents = [tokenized_sent + ['[SEP]'] for tokenized_sent in tokenized_sents] # tokenized_sents[0] = ['[CLS]'] + tokenized_sents[0] sent_align_para = [] last_idx = 0 for sent in range(len(tokenized_sents)): flag = False for _idx, para in enumerate(paras): if sent in para: sent_align_para.append(_idx) last_idx = _idx flag = True break if not flag: sent_align_para.append(last_idx) assert len(sent_align_para) == len(tokenized_sents) sent_align_para.append(last_idx + 1) segment_feat_para = [ sent_align_para[_sid] + 1 if sent_align_para[_sid] < MAX_FREQ - 1 else MAX_FREQ - 1 for _sid, sent in enumerate(tokenized_sents_2) for word in sent ] segment_feat_sent = [[ sent_align_para[_sid] + 1 if sent_align_para[_sid] < MAX_FREQ - 1 else MAX_FREQ - 1 for word in sent ] for _sid, sent in enumerate(tokenized_sents)] sent_align_para = [[_] for _ in sent_align_para] word_num = [len(tokenized_sent) for tokenized_sent in tokenized_sents] truncated_word_num = word_num # find out of range and useless nodes other_nodes = set() oor_nodes = [] # out of range nodes will not included in the graph word_freq_feat, word_inpara_feat, sent_freq_feat, sent_inpara_freq_feat = create_word_freq_in_para_feat( paras, tokenized_sents, tokenized_article) assert len(word_freq_feat) == len(tokenized_article) and len( word_inpara_feat) == len(tokenized_article) for _id, content in nodes.items(): words = [ pos for mention in content['content'] for pos in mention['word_pos'] if pos != -1 ] words = [word for word in words if word < max_len] if len(words) != 0: other_nodes.add(_id) else: oor_nodes.append(_id) activated_nodes = set() for _id, content in edges.items(): if content['content']['arg1'] not in oor_nodes and content['content'][ 'arg2'] not in oor_nodes: words = content['content']['word_pos'] new_words = [ word for word in words if word > -1 and word < max_len ] if len(new_words) > 0: activated_nodes.add(content['content']['arg1']) activated_nodes.add(content['content']['arg2']) oor_nodes.extend(list(other_nodes - activated_nodes)) # process nodes sorted_nodes = sorted(nodes.items(), key=lambda x: int(x[0].split('_')[1])) nodewords = [] nodefreq = [] sum_worthy = [] id2node = {} ii = 0 for _id, content in sorted_nodes: if _id not in oor_nodes: words = [ pos for mention in content['content'] for pos in mention['word_pos'] if pos != -1 ] words = [word for word in words if word < max_len] words = words[:node_max_len] sum_worthy.append(content[key]) if len(words) != 0: nodewords.append(words) nodefreq.append(len(content['content'])) id2node[_id] = ii ii += 1 else: oor_nodes.append(_id) if len(nodewords) == 0: # print('warning! no nodes in this sample') nodewords = [[0], [2]] nodefreq.extend([1, 1]) sum_worthy.extend([0, 0]) nodelength = [len(words) for words in nodewords] # process edges acticated_nodes = set() triples = [] edge_freq = [] relations = [] sum_worthy_edges = [] id2edge = {} sorted_edges = sorted(edges.items(), key=lambda x: int(x[0].split('_')[1])) ii = 0 for _id, content in sorted_edges: if content['content']['arg1'] not in oor_nodes and content['content'][ 'arg2'] not in oor_nodes: words = content['content']['word_pos'] new_words = [ word for word in words if word > -1 and word < max_len ] new_words = new_words[:node_max_len] if len(new_words) > 0: node1 = id2node[content['content']['arg1']] node2 = id2node[content['content']['arg2']] sum_worthy_edges.append(content[key]) if adj_type == 'edge_up': nodewords[node1].extend(new_words) elif adj_type == 'edge_down': nodewords[node2].extend(new_words) edge = int(_id.split('_')[1]) edge_freq.append(1) triples.append([node1, ii, node2]) acticated_nodes.add(content['content']['arg1']) acticated_nodes.add(content['content']['arg2']) id2edge[_id] = ii ii += 1 relations.append(new_words) if len(relations) == 0: # print('warning! no edges in this sample') relations = [[1]] edge_freq = [1] triples = [[0, 0, 1]] sum_worthy_edges.extend([0]) node_lists = [] edge_lists = [] triples = [] if max_sent is None: max_sent = 9999 for _sgid, subgraph in enumerate(subgraphs): try: paraid = paras[_sgid][0] except: paraid = 0 if type(paraid) != type(max_sent): paraid = 0 if paraid > max_sent - 1: continue if subgraph == []: node_lists.append([]) triples.append([]) edge_lists.append([]) else: node_list = set() triple = [] edge_list = [] eidx = [] for _triple in subgraph: if _triple[0] not in oor_nodes and _triple[ 2] not in oor_nodes and id2edge.__contains__( _triple[1]): node_list.add(id2node[_triple[0]]) node_list.add(id2node[_triple[2]]) eidx.append(_triple[1]) node_list = list(sorted(node_list)) for _triple in subgraph: if _triple[0] not in oor_nodes and _triple[ 2] not in oor_nodes and id2edge.__contains__( _triple[1]): idx1 = node_list.index(id2node[_triple[0]]) idx2 = node_list.index(id2node[_triple[2]]) _idxe = id2edge[_triple[1]] idxe_in_subgraph = eidx.index(_triple[1]) edge_list.append(_idxe) triple.append([idx1, idxe_in_subgraph, idx2]) triples.append(triple) node_lists.append(node_list) edge_lists.append(edge_list) if len(node_lists) == 0: node_lists.append([]) triples.append([]) edge_lists.append([]) rlength = [len(words) for words in relations] nodefreq = [ freq if freq < MAX_FREQ - 1 else MAX_FREQ - 1 for freq in nodefreq ] if adj_type == 'edge_as_node': node_num = len(nodewords) nodewords = nodewords + relations nodefreq = nodefreq + edge_freq nodelength = nodelength + rlength sum_worthy = sum_worthy + sum_worthy_edges for i in range(len(triples)): node_lists[i] = node_lists[i] + [ edge + node_num for edge in edge_lists[i] ] gold_dec_selection_label = [0 for i in range(len(node_lists))] for sent in cleaned_extracts: for i, para in enumerate(paras): if sent in para: gold_dec_selection_label[i] = 1 return tokenized_article, truncated_word_num, ( nodewords, sum_worthy, gold_dec_selection_label), (relations, triples, node_lists, sent_align_para, segment_feat_sent, segment_feat_para, nodefreq, word_freq_feat, word_inpara_feat, sent_freq_feat, sent_inpara_freq_feat)
def decode(save_path, abs_dir, ext_dir, split, batch_size, max_len, cuda, trans=False): start = time() # setup model if abs_dir is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization abstractor = identity else: abstractor = Abstractor(abs_dir, max_len, cuda) if ext_dir is None: # NOTE: if no abstractor is provided then # it would be the lead-N extractor extractor = lambda art_sents: list(range(len(art_sents)))[:MAX_ABS_NUM] else: extractor = Extractor(ext_dir, max_ext=MAX_ABS_NUM, cuda=cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll ) # prepare save paths and logs for i in range(MAX_ABS_NUM): os.makedirs(join(save_path, 'output_{}'.format(i))) # os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['abstractor'] = (None if abs_dir is None else json.load(open(join(abs_dir, 'meta.json')))) dec_log['extractor'] = (None if ext_dir is None else json.load(open(join(ext_dir, 'meta.json')))) dec_log['rl'] = False dec_log['split'] = split dec_log['beam'] = 1 # greedy decoding only with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): if trans: tokenized_article_batch = raw_article_batch # else: tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: if trans: ext, batch = extractor(raw_art_sents) art_sents = batch.src_str[0] # print(ext, [x.nonzero(as_tuple=True)[0] for x in batch.src_sent_labels]) for k, idx in enumerate([ext]): _pred = [] _ids = [] if (len(batch.src_str[k]) == 0): continue for j in idx[:min(len(ext), len(batch.src_str[k]))]: if (j >= len(batch.src_str[k])): continue candidate = batch.src_str[k][j].strip() if (not _block_tri(candidate, _pred)): _pred.append(candidate) _ids.append(j) else: continue if (len(_pred) == 3): break # print(ext, _ids, [x.nonzero(as_tuple=True)[0] for x in batch.src_sent_labels], list(map(lambda i: art_sents[i], ext))) ext = _ids ext_inds += [(len(ext_arts), len(ext))] ext_arts += list(map(lambda i: art_sents[i], ext)) else: ext = extractor(raw_art_sents) ext_inds += [(len(ext_arts), len(ext))] ext_arts += list(map(lambda i: raw_art_sents[i], ext)) dec_outs = abstractor(ext_arts) # print(dec_outs) assert i == batch_size*i_debug for j, n in ext_inds: if trans: decoded_sents = dec_outs[j:j+n] else: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j+n]] for k, dec_str in enumerate(decoded_sents): with open(join(save_path, 'output_{}/{}.dec'.format(k, i)), 'w') as f: f.write(make_html_safe(dec_str)) #f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i/n_data*100, timedelta(seconds=int(time()-start)) ), end='') # if i_debug == 1: # break print()
def decode(save_path, model_dir, split, batch_size, beam_size, diverse, max_len, cuda): start = time() # setup model with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = identity else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda) extractor = RLExtractor(model_dir, cuda=cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll ) # prepare save paths and logs os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['abstractor'] = meta['net_args']['abstractor'] dec_log['extractor'] = meta['net_args']['extractor'] dec_log['rl'] = True dec_log['split'] = split dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents)[:-1] # exclude EOE if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] else: ext = [i.item() for i in ext] ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] if beam_size > 1: all_beams = abstractor(ext_arts, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) else: dec_outs = abstractor(ext_arts) assert i == batch_size*i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j+n]] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i/n_data*100, timedelta(seconds=int(time()-start)) ), end='') print()
def decode(save_path, model_dir, split, batch_size, beam_size, diverse, max_len, cuda, sc, min_len): start = time() # setup model with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: #if not meta['net_args'].__contains__('abstractor'): # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = identity else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda, min_len) if sc: extractor = SCExtractor(model_dir, cuda=cuda) else: extractor = RLExtractor(model_dir, cuda=cuda) #check if use bert try: _bert = extractor._net._bert except: _bert = False print('no bert arg:') if _bert: tokenizer = BertTokenizer.from_pretrained( 'bert-large-uncased-whole-word-masking') print('bert tokenizer loaded') # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll) # prepare save paths and logs os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['abstractor'] = meta['net_args']['abstractor'] dec_log['extractor'] = meta['net_args']['extractor'] dec_log['rl'] = True dec_log['split'] = split dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding if sc: i = 0 length = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] if _bert: for raw_art_sents, raw_art in zip(tokenized_article_batch, raw_article_batch): tokenized_sents = [ tokenizer.tokenize(source_sent.lower()) for source_sent in raw_art ] tokenized_sents = [ tokenized_sent + ['[SEP]'] for tokenized_sent in tokenized_sents ] tokenized_sents[0] = ['[CLS]'] + tokenized_sents[0] word_num = [ len(tokenized_sent) for tokenized_sent in tokenized_sents ] truncated_word_num = [] total_count = 0 for num in word_num: if total_count + num < MAX_LEN_BERT: truncated_word_num.append(num) else: truncated_word_num.append(MAX_LEN_BERT - total_count) break total_count += num tokenized_sents = list( concat(tokenized_sents))[:MAX_LEN_BERT] tokenized_sents = tokenizer.convert_tokens_to_ids( tokenized_sents) art_sents = tokenize(None, raw_art) _input = (art_sents, tokenized_sents, truncated_word_num) ext = extractor(_input)[:] # exclude EOE if not ext: # use top-3 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(3))[:len(raw_art_sents)] else: ext = [i for i in ext] ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] else: for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents)[:] # exclude EOE if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] else: ext = [i for i in ext] ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] if beam_size > 1: all_beams = abstractor(ext_arts, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) else: dec_outs = abstractor(ext_arts) assert i == batch_size * i_debug for j, n in ext_inds: decoded_sents = [ ' '.join(dec) for dec in dec_outs[j:j + n] ] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') length += len(decoded_sents) else: i = 0 length = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents)[:-1] # exclude EOE if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] else: ext = [i.item() for i in ext] ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] if beam_size > 1: all_beams = abstractor(ext_arts, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) else: dec_outs = abstractor(ext_arts) assert i == batch_size * i_debug for j, n in ext_inds: decoded_sents = [ ' '.join(dec) for dec in dec_outs[j:j + n] ] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') length += len(decoded_sents) print('average summary length:', length / i)
def decode_graph(save_path, model_dir, split, batch_size, beam_size, diverse, max_len, cuda, sc, min_len, docgraph, paragraph): start = time() # setup model with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: #if not meta['net_args'].__contains__('abstractor'): # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = identity else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda, min_len=min_len) print('docgraph:', docgraph) extractor = SCExtractor(model_dir, cuda=cuda, docgraph=docgraph, paragraph=paragraph) adj_type = extractor._net._adj_type bert = extractor._net._bert if bert: tokenizer = extractor._net._bert try: with open( '/data/luyang/process-nyt/bert_tokenizaiton_aligns/robertaalign-base-cased.pkl', 'rb') as f: align = pickle.load(f) except FileNotFoundError: with open( '/data2/luyang/process-nyt/bert_tokenizaiton_aligns/robertaalign-base-cased.pkl', 'rb') as f: align = pickle.load(f) try: with open( '/data/luyang/process-cnn-dailymail/bert_tokenizaiton_aligns/robertaalign-base-cased.pkl', 'rb') as f: align2 = pickle.load(f) except FileNotFoundError: with open( '/data2/luyang/process-cnn-dailymail/bert_tokenizaiton_aligns/robertaalign-base-cased.pkl', 'rb') as f: align2 = pickle.load(f) align.update(align2) # setup loader def coll(batch): batch = list(filter(bool, batch)) return batch dataset = DecodeDatasetGAT(split, args.key) n_data = len(dataset) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll) # prepare save paths and logs os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['abstractor'] = meta['net_args']['abstractor'] dec_log['extractor'] = meta['net_args']['extractor'] dec_log['rl'] = True dec_log['split'] = split dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 length = 0 sent_selected = 0 with torch.no_grad(): for i_debug, raw_input_batch in enumerate(loader): raw_article_batch, nodes, edges, paras, subgraphs = zip( *raw_input_batch) if bert: art_sents = [[ tokenizer.tokenize(source_sent) for source_sent in source_sents ] for source_sents in raw_article_batch] for _i in range(len(art_sents)): art_sents[_i][0] = [tokenizer.bos_token] + art_sents[_i][0] art_sents[_i][-1] = art_sents[_i][-1] + [ tokenizer.eos_token ] truncated_word_nums = [] word_nums = [[len(sent) for sent in art_sent] for art_sent in art_sents] for word_num in word_nums: truncated_word_num = [] total_count = 0 for num in word_num: if total_count + num < args.max_dec_word: truncated_word_num.append(num) else: truncated_word_num.append(args.max_dec_word - total_count) break total_count += num truncated_word_nums.append(truncated_word_num) sources = [ list(concat(art_sent))[:args.max_dec_word] for art_sent in art_sents ] else: tokenized_article_batch = map(tokenize(None), raw_article_batch) #processed_clusters = map(preproc(list(tokenized_article_batch), clusters)) #processed_clusters = list(zip(*processed_clusters)) ext_arts = [] ext_inds = [] pre_abs = [] beam_inds = [] if bert: for raw_art_sents, source, art_sent, word_num, raw_nodes, raw_edges, raw_paras, raw_subgraphs in zip( raw_article_batch, sources, art_sents, truncated_word_nums, nodes, edges, paras, subgraphs): processed_nodes = prepro_rl_graph_bert( align, raw_art_sents, source, art_sent, args.max_dec_word, raw_nodes, raw_edges, raw_paras, raw_subgraphs, adj_type, docgraph) _input = (raw_art_sents, source) + processed_nodes + (word_num, ) ext = extractor(_input)[:] sent_selected += len(ext) if not ext: # use top-3 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(3))[:len(raw_art_sents)] else: ext = [i for i in ext] ext_art = list(map(lambda i: raw_art_sents[i], ext)) pre_abs.append([word for sent in ext_art for word in sent]) beam_inds += [(len(beam_inds), 1)] else: for raw_art_sents, raw_nodes, raw_edges, raw_paras, raw_subgraphs in zip( tokenized_article_batch, nodes, edges, paras, subgraphs): processed_nodes = prepro_rl_graph(raw_art_sents, raw_nodes, raw_edges, raw_paras, raw_subgraphs, adj_type, docgraph) _input = (raw_art_sents, ) + processed_nodes ext = extractor(_input)[:] # exclude EOE sent_selected += len(ext) if not ext: # use top-3 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(3))[:len(raw_art_sents)] else: ext = [i for i in ext] ext_art = list(map(lambda i: raw_art_sents[i], ext)) pre_abs.append([word for sent in ext_art for word in sent]) beam_inds += [(len(beam_inds), 1)] if beam_size > 1: # all_beams = abstractor(ext_arts, beam_size, diverse) # dec_outs = rerank_mp(all_beams, ext_inds) all_beams = abstractor(pre_abs, beam_size, diverse=1.0) dec_outs = rerank_mp(all_beams, beam_inds) else: dec_outs = abstractor(pre_abs) for dec_out in dec_outs: dec_out = sent_tokenize(' '.join(dec_out)) ext = [sent.split(' ') for sent in dec_out] ext_inds += [(len(ext_arts), len(ext))] ext_arts += ext dec_outs = ext_arts assert i == batch_size * i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j + n]] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') length += len(decoded_sents) print('average summary length:', length / i) print('average sentence selected:', sent_selected)
def decode(args, predict=False): # save_path = args.path batch_size = args.batch beam_size = args.beam diverse = args.div start = time() extractor = args.extractor abstractor = args.abstractor # setup model text = '' # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles if not predict: dataset = DecodeDataset(args) n_data = len(dataset) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll) else: n_data = 1 loader = clean_and_split(args.text) loader = [[[' '.join(mecab_tokenizer(line)) for line in loader]]] text = '\n'.join(loader[0][0]) i = 0 #print(text) with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents)[:-1] # exclude EOE if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] else: ext = [i.item() for i in ext] ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] if beam_size > 1: #print(ext_arts) all_beams = abstractor(ext_arts, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) else: dec_outs = abstractor(ext_arts) assert i == batch_size * i_debug source_text = [''.join(sent) for sent in ext_arts] for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j + n]] decoded_sents = decoded_sents[:20] # with open(join(save_path, 'output/{}.dec'.format(i)), # 'w') as f: # f.write(make_html_safe('\n'.join(decoded_sents))) result = make_html_safe('\n\n'.join(decoded_sents)) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') print() return text, result, source_text
def coll(tokenizer, align, max_src_len, adj_type, batch): art_batch, abs_batch, all_nodes, all_edges, all_subgraphs, all_paras = unzip( batch) def is_good_data(d): """ make sure data is not empty""" source_sents, extracts, nodes, edges, subgraphs, paras = d return source_sents and extracts and nodes and subgraphs and paras art_batch, abs_batch, all_nodes, all_edges, all_subgraphs, all_paras = list( zip(*list( filter( is_good_data, zip(art_batch, abs_batch, all_nodes, all_edges, all_subgraphs, all_paras))))) old_sources = art_batch art_sents = [[ tokenizer.tokenize(source_sent) for source_sent in source_sents ] for source_sents in art_batch] for _i in range(len(art_sents)): art_sents[_i][0] = [tokenizer.bos_token] + art_sents[_i][0] art_sents[_i][-1] = art_sents[_i][-1] + [tokenizer.eos_token] truncated_word_nums = [] word_nums = [[len(sent) for sent in art_sent] for art_sent in art_sents] for word_num in word_nums: truncated_word_num = [] total_count = 0 for num in word_num: if total_count + num < max_src_len: truncated_word_num.append(num) else: truncated_word_num.append(max_src_len - total_count) break total_count += num truncated_word_nums.append(truncated_word_num) sources = [ list(concat(art_sent))[:max_src_len] for art_sent in art_sents ] raw_art_sents = list(filter(bool, map(tokenize(None), art_batch))) abs_sents = list(filter(bool, map(tokenize(None), abs_batch))) max_sents = list( map(count_max_sent(max_source_num=max_src_len), art_sents)) inputs = [] # merge cluster nodewords, nodelength, nodefreq, sum_worthy, triples, relations = \ list(zip(*[process_nodes_bert(align, node, edge, len(source) - 1, max_sent, key='InSalientSent', adj_type=adj_type, source_sent=sent, paras=para, subgraphs=subgraph, docgraph=docgraph, source=source) for node, edge, sent, para, subgraph, source, max_sent in zip(all_nodes, all_edges, old_sources, all_paras, all_subgraphs, sources, max_sents)])) # for art_sent, nodes, edges, subgraphs, paras in zip(art_sents, all_nodes, all_edges, all_subgraphs, all_paras): # max_len = len(list(concat(art_sent))) # # nodewords, nodelength, nodefreq, sum_worthy, triples, relations, sent_node_aligns = process_nodes(nodes, edges, max_len, max_sent_num=len(list(art_sent)), key=key, adj_type=adj_type) # if paragraph: # nodewords, node_lists, nodefreq, sum_worthy, triples, relations = process_subgraphs( # nodes, edges, subgraphs, paras, max_len, max_sent=len(list(art_sent)), # key=key, adj_type=adj_type # ) # sent_align_para = [] # last_idx = 0 # for sent in range(len(art_sent)): # flag = False # for _idx, para in enumerate(paras): # if sent in para: # sent_align_para.append([_idx]) # last_idx = _idx # flag = True # break # if not flag: # sent_align_para.append([last_idx]) # assert len(sent_align_para) == len(art_sent) # sent_align_para.append([last_idx + 1]) # if docgraph: # inputs.append((art_sent, nodewords, nodefreq, triples, relations, sent_node_aligns)) # elif paragraph: # inputs.append((art_sent, nodewords, nodefreq, triples, # relations, sent_align_para, node_lists)) # else: # raise Exception('wrong graph type') if docgraph: inputs = list( zip(raw_art_sents, sources, nodewords, nodefreq, triples, relations, truncated_word_nums)) else: node_lists = nodelength inputs = list( zip(raw_art_sents, sources, nodewords, nodefreq, triples, relations, node_lists, truncated_word_nums)) assert len(inputs) == len(abs_sents) return inputs, abs_sents
def coll(key, adj_type, batch): split_token = '<split>' pad = 0 art_batch, abs_batch, all_nodes, all_edges, all_subgraphs, all_paras = unzip( batch) def is_good_data(d): """ make sure data is not empty""" source_sents, extracts, nodes, edges, subgraphs, paras = d return source_sents and extracts and nodes and subgraphs and paras art_batch, abs_batch, all_nodes, all_edges, all_subgraphs, all_paras = list( zip(*list( filter( is_good_data, zip(art_batch, abs_batch, all_nodes, all_edges, all_subgraphs, all_paras))))) art_sents = list(filter(bool, map(tokenize(None), art_batch))) abs_sents = list(filter(bool, map(tokenize(None), abs_batch))) inputs = [] # merge cluster for art_sent, nodes, edges, subgraphs, paras in zip( art_sents, all_nodes, all_edges, all_subgraphs, all_paras): max_len = len(list(concat(art_sent))) _, word_inpara_freq_feat, _, sent_inpara_freq_feat = create_word_freq_in_para_feat( paras, art_sent, list(concat(art_sent))) nodewords, nodelength, nodefreq, sum_worthy, triples, relations, sent_node_aligns = process_nodes( nodes, edges, max_len, max_sent_num=len(list(art_sent)), key=key, adj_type=adj_type) if paragraph: nodewords, node_lists, nodefreq, sum_worthy, triples, relations = process_subgraphs( nodes, edges, subgraphs, paras, max_len, max_sent=len(list(art_sent)), key=key, adj_type=adj_type) sent_align_para = [] last_idx = 0 for sent in range(len(art_sent)): flag = False for _idx, para in enumerate(paras): if sent in para: sent_align_para.append([_idx]) last_idx = _idx flag = True break if not flag: sent_align_para.append([last_idx]) assert len(sent_align_para) == len(art_sent) sent_align_para.append([last_idx + 1]) if docgraph: inputs.append((art_sent, nodewords, nodefreq, word_inpara_freq_feat, sent_inpara_freq_feat, triples, relations, sent_node_aligns)) elif paragraph: inputs.append( (art_sent, nodewords, nodefreq, word_inpara_freq_feat, sent_inpara_freq_feat, triples, relations, sent_align_para, node_lists)) else: raise Exception('wrong graph type') assert len(inputs) == len(abs_sents) return inputs, abs_sents
def decode(save_path, abs_dir, ext_dir, split, batch_size, max_len, cuda): start = time() # setup model if abs_dir is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization abstractor = identity else: abstractor = Abstractor(abs_dir, max_len, cuda) if ext_dir is None: # NOTE: if no abstractor is provided then # it would be the lead-N extractor extractor = lambda art_sents: list(range(len(art_sents)))[:MAX_ABS_NUM] else: extractor = Extractor(ext_dir, max_ext=MAX_ABS_NUM, cuda=cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll ) # prepare save paths and logs for i in range(MAX_ABS_NUM): os.makedirs(join(save_path, 'output_{}'.format(i))) dec_log = {} dec_log['abstractor'] = (None if abs_dir is None else json.load(open(join(abs_dir, 'meta.json')))) dec_log['extractor'] = (None if ext_dir is None else json.load(open(join(ext_dir, 'meta.json')))) dec_log['rl'] = False dec_log['split'] = split dec_log['beam'] = 1 # greedy decoding only with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents) ext_inds += [(len(ext_arts), len(ext))] ext_arts += list(map(lambda i: raw_art_sents[i], ext)) dec_outs = abstractor(ext_arts) assert i == batch_size*i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j+n]] for k, dec_str in enumerate(decoded_sents): with open(join(save_path, 'output_{}/{}.dec'.format(k, i)), 'w') as f: f.write(make_html_safe(dec_str)) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i/n_data*100, timedelta(seconds=int(time()-start)) ), end='') print()
def coll(batch): art_batch, abs_batch, extracted = unzip(batch) art_sents = list(filter(bool, map(tokenize(None), art_batch))) abs_sents = list(filter(bool, map(tokenize(None), abs_batch))) extracted = list(filter(bool, extracted)) return art_sents, abs_sents, extracted
def decode(save_path, model_dir, split, batch_size, beam_size, diverse, max_len, cuda): start = time() # setup model with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = identity else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda) extractor = RLExtractor(model_dir, cuda=cuda) # setup loader def coll(batch): articles, abstract, extracted = unzip(batch) articles = list(filter(bool, articles)) abstract = list(filter(bool, abstract)) extracted = list(filter(bool, extracted)) return articles, abstract, extracted dataset = DecodeDataset(split) n_data = len(dataset[0]) # article sentence loader = DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll ) # prepare save paths and logs if os.path.exists(join(save_path, 'output')): pass else: os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['abstractor'] = meta['net_args']['abstractor'] dec_log['extractor'] = meta['net_args']['extractor'] dec_log['rl'] = True dec_log['split'] = split dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) file_path = os.path.join(save_path, 'Attention') act_path = os.path.join(save_path, 'Actions') header = "index,rouge_score1,rouge_score2,"+\ "rouge_scorel,dec_sent_nums,abs_sent_nums,doc_sent_nums,doc_words_nums,"+\ "ext_words_nums, abs_words_nums, diff,"+\ "recall, precision, less_rewrite, preserve_action, rewrite_action, each_actions,"+\ "top3AsAns, top3AsGold, any_top2AsAns, any_top2AsGold,true_rewrite,true_preserve\n" if not os.path.exists(file_path): print('create dir:{}'.format(file_path)) os.makedirs(file_path) if not os.path.exists(act_path): print('create dir:{}'.format(act_path)) os.makedirs(act_path) with open(join(save_path,'_statisticsDecode.log.csv'),'w') as w: w.write(header) # Decoding i = 0 with torch.no_grad(): for i_debug, (raw_article_batch, raw_abstract_batch, extracted_batch) in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) tokenized_abstract_batch = map(tokenize(None), raw_abstract_batch) token_nums_batch = list(map(token_nums(None), raw_article_batch)) ext_nums = [] ext_arts = [] ext_inds = [] rewrite_less_rouge = [] dec_outs_act = [] ext_acts = [] abs_collections = [] ext_collections = [] # 抽句子 for ind, (raw_art_sents, abs_sents) in enumerate(zip(tokenized_article_batch ,tokenized_abstract_batch)): (ext, (state, act_dists)), act = extractor(raw_art_sents) # exclude EOE extracted_state = state[extracted_batch[ind]] attn = torch.softmax(state.mm(extracted_state.transpose(1,0)),dim=-1) # (_, abs_state), _ = extractor(abs_sents) # exclude EOE def plot_actDist(actons, nums): print('indiex: {} distribution ...'.format(nums)) # Write MDP State Attention weight matrix file_name = os.path.join(act_path, '{}.attention.pdf'.format(nums)) pdf_pages = PdfPages(file_name) plot_attention(actons.cpu().numpy(), name='{}-th artcle'.format(nums), X_label=list(range(len(raw_art_sents))), Y_label=list(range(len(ext))), dirpath=save_path, pdf_page=pdf_pages,action=True) pdf_pages.close() # plot_actDist(torch.stack(act_dists, dim=0), nums=ind+i) def plot_attn(): print('indiex: {} write_attention_pdf ...'.format(i + ind)) # Write MDP State Attention weight matrix file_name = os.path.join(file_path, '{}.attention.pdf'.format(i+ind)) pdf_pages = PdfPages(file_name) plot_attention(attn.cpu().numpy(), name='{}-th artcle'.format(i+ind), X_label=extracted_batch[ind],Y_label=list(range(len(raw_art_sents))), dirpath=save_path, pdf_page=pdf_pages) pdf_pages.close() # plot_attn() ext = ext[:-1] act = act[:-1] if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] act = list([1]*5)[:len(raw_art_sents)] else: ext = [i.item() for i in ext] act = [i.item() for i in act] ext_nums.append(ext) ext_inds += [(len(ext_arts), len(ext))] # [(0,5),(5,7),(7,3),...] ext_arts += [raw_art_sents[k] for k in ext] ext_acts += [k for k in act] # 計算累計的句子 ext_collections += [sum(ext_arts[ext_inds[-1][0]:ext_inds[-1][0]+k+1],[]) for k in range(ext_inds[-1][1])] abs_collections += [sum(abs_sents[:k+1],[]) if k<len(abs_sents) else sum(abs_sents[0:len(abs_sents)],[]) for k in range(ext_inds[-1][1])] if beam_size > 1: # do n times abstract all_beams = abstractor(ext_arts, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) dec_collections = [[sum(dec_outs[pos[0]:pos[0]+k+1],[]) for k in range(pos[1])] for pos in ext_inds] dec_collections = [x for sublist in dec_collections for x in sublist] for index, chooser in enumerate(ext_acts): if chooser == 0: dec_outs_act += [dec_outs[index]] else: dec_outs_act += [ext_arts[index]] assert len(ext_collections)==len(dec_collections)==len(abs_collections) for ext, dec, abss, act in zip(ext_collections, dec_collections, abs_collections, ext_acts): # for each sent in extracted digest # All abstract mapping rouge_before_rewriten = compute_rouge_n(ext, abss, n=1) rouge_after_rewriten = compute_rouge_n(dec, abss, n=1) diff_ins = rouge_before_rewriten - rouge_after_rewriten rewrite_less_rouge.append(diff_ins) else: # do 1st abstract dec_outs = abstractor(ext_arts) dec_collections = [[sum(dec_outs[pos[0]:pos[0]+k+1],[]) for k in range(pos[1])] for pos in ext_inds] dec_collections = [x for sublist in dec_collections for x in sublist] for index, chooser in enumerate(ext_acts): if chooser == 0: dec_outs_act += [dec_outs[index]] else: dec_outs_act += [ext_arts[index]] # dec_outs_act = dec_outs # dec_outs_act = ext_arts assert len(ext_collections)==len(dec_collections)==len(abs_collections) for ext, dec, abss, act in zip(ext_collections, dec_collections, abs_collections, ext_acts): # for each sent in extracted digest # All abstract mapping rouge_before_rewriten = compute_rouge_n(ext, abss, n=1) rouge_after_rewriten = compute_rouge_n(dec, abss, n=1) diff_ins = rouge_before_rewriten - rouge_after_rewriten rewrite_less_rouge.append(diff_ins) assert i == batch_size*i_debug for iters, (j, n) in enumerate(ext_inds): do_right_rewrite = sum([1 for rouge, action in zip(rewrite_less_rouge[j:j+n], ext_acts[j:j+n]) if rouge<0 and action==0]) do_right_preserve = sum([1 for rouge, action in zip(rewrite_less_rouge[j:j+n], ext_acts[j:j+n]) if rouge>=0 and action==1]) decoded_words_nums = [len(dec) for dec in dec_outs_act[j:j+n]] ext_words_nums = [token_nums_batch[iters][x] for x in range(len(token_nums_batch[iters])) if x in ext_nums[iters]] # 皆取extracted label # decoded_sents = [raw_article_batch[iters][x] for x in extracted_batch[iters]] # 統計數據 [START] decoded_sents = [' '.join(dec) for dec in dec_outs_act[j:j+n]] rouge_score1 = compute_rouge_n(' '.join(decoded_sents),' '.join(raw_abstract_batch[iters]),n=1) rouge_score2 = compute_rouge_n(' '.join(decoded_sents),' '.join(raw_abstract_batch[iters]),n=2) rouge_scorel = compute_rouge_l(' '.join(decoded_sents),' '.join(raw_abstract_batch[iters])) dec_sent_nums = len(decoded_sents) abs_sent_nums = len(raw_abstract_batch[iters]) doc_sent_nums = len(raw_article_batch[iters]) doc_words_nums = sum(token_nums_batch[iters]) ext_words_nums = sum(ext_words_nums) abs_words_nums = sum(decoded_words_nums) label_recall = len(set(ext_nums[iters]) & set(extracted_batch[iters])) / len(extracted_batch[iters]) label_precision = len(set(ext_nums[iters]) & set(extracted_batch[iters])) / len(ext_nums[iters]) less_rewrite = rewrite_less_rouge[j+n-1] dec_one_action_num = sum(ext_acts[j:j+n]) dec_zero_action_num = n - dec_one_action_num ext_indices = '_'.join([str(i) for i in ext_nums[iters]]) top3 = set([0,1,2]) <= set(ext_nums[iters]) top3_gold = set([0,1,2]) <= set(extracted_batch[iters]) # Any Top 2 top2 = set([0,1]) <= set(ext_nums[iters]) or set([1,2]) <= set(ext_nums[iters]) or set([0,2]) <= set(ext_nums[iters]) top2_gold = set([0,1]) <= set(extracted_batch[iters]) or set([1,2]) <= set(extracted_batch[iters]) or set([0,2]) <= set(extracted_batch[iters]) with open(join(save_path,'_statisticsDecode.log.csv'),'a') as w: w.write('{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}\n'.format(i,rouge_score1, rouge_score2, rouge_scorel, dec_sent_nums, abs_sent_nums, doc_sent_nums, doc_words_nums, ext_words_nums,abs_words_nums,(ext_words_nums - abs_words_nums), label_recall, label_precision, less_rewrite, dec_one_action_num, dec_zero_action_num, ext_indices, top3, top3_gold, top2, top2_gold,do_right_rewrite,do_right_preserve)) # 統計數據 END with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: decoded_sents = [i for i in decoded_sents if i!=''] if len(decoded_sents) > 0: f.write(make_html_safe('\n'.join(decoded_sents))) else: f.write('') i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i/n_data*100, timedelta(seconds=int(time()-start)) ), end='') print()
def prepro_gat_nobert(batch, max_sent_len=100, max_sent=60, node_max_len=30, key='summary_worthy', adj_type='concat_triple'): source_sents, nodes, edges, paras = batch tokenized_sents = tokenize(max_sent_len, source_sents)[:max_sent] tokenized_sents_2 = tokenize(None, source_sents)[:max_sent] tokenized_article = list(concat(tokenized_sents_2)) max_len = len(tokenized_article) # tokenized_sents = [tokenized_sent + ['[SEP]'] for tokenized_sent in tokenized_sents] # tokenized_sents[0] = ['[CLS]'] + tokenized_sents[0] word_num = [len(tokenized_sent) for tokenized_sent in tokenized_sents] truncated_word_num = word_num # find out of range and useless nodes other_nodes = set() oor_nodes = [] # out of range nodes will not included in the graph word_freq_feat, word_inpara_feat, sent_freq_feat, sent_inpara_freq_feat = create_word_freq_in_para_feat( paras, tokenized_sents, tokenized_article) assert len(word_freq_feat) == len(tokenized_article) and len( word_inpara_feat) == len(tokenized_article) for _id, content in nodes.items(): words = [ pos for mention in content['content'] for pos in mention['word_pos'] if pos != -1 ] words = [word for word in words if word < max_len] if len(words) != 0: other_nodes.add(_id) else: oor_nodes.append(_id) activated_nodes = set() for _id, content in edges.items(): if content['content']['arg1'] not in oor_nodes and content['content'][ 'arg2'] not in oor_nodes: words = content['content']['word_pos'] new_words = [ word for word in words if word > -1 and word < max_len ] if len(new_words) > 0: activated_nodes.add(content['content']['arg1']) activated_nodes.add(content['content']['arg2']) oor_nodes.extend(list(other_nodes - activated_nodes)) # process nodes sorted_nodes = sorted(nodes.items(), key=lambda x: int(x[0].split('_')[1])) nodewords = [] nodefreq = [] nodeinsent = [] nodetype = [] sum_worthy = [] id2node = {} ii = 0 for _id, content in sorted_nodes: if _id not in oor_nodes: words = [ pos for mention in content['content'] for pos in mention['word_pos'] if pos != -1 ] words = [word for word in words if word < max_len] words = words[:node_max_len] sum_worthy.append(content[key]) if len(words) != 0: nodewords.append(words) nodefreq.append(len(content['content'])) nodetype.append(1) nodeinsent.append([ mention['sent_pos'] for mention in content['content'] if mention['sent_pos'] < len(tokenized_sents) ]) id2node[_id] = ii ii += 1 else: oor_nodes.append(_id) if len(nodewords) == 0: # print('warning! no nodes in this sample') nodewords = [[0], [2]] nodefreq.extend([1, 1]) nodeinsent.extend([[0], [0]]) nodetype.extend([1, 1]) sum_worthy.extend([0, 0]) nodelength = [len(words) for words in nodewords] # process edges acticated_nodes = set() triples = [] edge_freq = [] edgeinsent = [] edgetype = [] relations = [] sum_worthy_edges = [] sorted_edges = sorted(edges.items(), key=lambda x: int(x[0].split('_')[1])) ii = 0 for _id, content in sorted_edges: if content['content']['arg1'] not in oor_nodes and content['content'][ 'arg2'] not in oor_nodes: words = content['content']['word_pos'] new_words = [ word for word in words if word > -1 and word < max_len ] new_words = new_words[:node_max_len] if len(new_words) > 0: node1 = id2node[content['content']['arg1']] node2 = id2node[content['content']['arg2']] sum_worthy_edges.append(content[key]) try: sent_pos = [content['content']['sent_pos']] except KeyError: sent_pos = [ content['content']['arg1_original'][0]['sent_pos'] ] if adj_type == 'edge_up': nodewords[node1].extend(new_words) elif adj_type == 'edge_down': nodewords[node2].extend(new_words) edge = int(_id.split('_')[1]) edge_freq.append(1) edgeinsent.append(sent_pos) edgetype.append(2) triples.append([node1, ii, node2]) acticated_nodes.add(content['content']['arg1']) acticated_nodes.add(content['content']['arg2']) ii += 1 relations.append(new_words) if len(relations) == 0: # print('warning! no edges in this sample') relations = [[1]] triples = [[0, 0, 1]] edgeinsent.append([0]) edge_freq = [1] edgetype.append(2) sum_worthy_edges.extend([0]) nodefreq = [ freq if freq < MAX_FREQ - 1 else MAX_FREQ - 1 for freq in nodefreq ] rlength = [len(words) for words in relations] if adj_type == 'edge_as_node': nodewords = nodewords + relations nodelength = nodelength + rlength sum_worthy = sum_worthy + sum_worthy_edges nodefreq = nodefreq + edge_freq nodetype = nodetype + edgetype nodeinsent = nodeinsent + edgeinsent sent_node_aligns = create_sent_node_align(nodeinsent, len(tokenized_sents)) return tokenized_article, truncated_word_num, (nodewords, nodelength, sum_worthy, nodefreq, word_freq_feat, word_inpara_feat, sent_freq_feat, sent_inpara_freq_feat, sent_node_aligns), \ (relations, rlength, triples)
def decode(save_path, model_dir, split, batch_size, beam_size, diverse, max_len, cuda): start = time() # setup model with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = lambda x,y:x else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: print('BEAM') abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda) extractor = RLExtractor(model_dir, cuda=cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll ) # prepare save paths and logs try: os.makedirs(join(save_path, 'output')) except: pass dec_log = {} dec_log['abstractor'] = meta['net_args']['abstractor'] dec_log['extractor'] = meta['net_args']['extractor'] dec_log['rl'] = True dec_log['split'] = split dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 total_leng = 0 total_num = 0 with torch.no_grad(): for i_debug, data_batch in enumerate(loader): raw_article_batch, sent_label_batch = tuple(map(list, unzip(data_batch))) tokenized_article_batch = map(tokenize(None), raw_article_batch) #ext_arts = [] ext_inds = [] dirty = [] ext_sents = [] masks = [] for raw_art_sents, sent_labels in zip(tokenized_article_batch, sent_label_batch): ext = extractor(raw_art_sents, sent_labels) # exclude EOE tmp_size = min(max_dec_edu, len(ext) - 1) #total_leng += sum([len(e) -1 for e in ext[:-1]]) #total_num += len(ext) - 1 #print(tmp_size, len(ext) - 1) ext_inds += [(len(ext_sents), tmp_size)] tmp_stop = ext[-1][-1].item() tmp_truncate = tmp_stop - 1 str_arts = list(map(lambda x: ' '.join(x), raw_art_sents)) for idx in ext[:tmp_size]: t, m = rl_edu_to_sentence(str_arts, idx) total_leng += len(t) total_num += 1 assert len(t) == len(m) if t == []: assert len(idx) == 1 id = idx[0].item() if id == tmp_truncate: dirty.append(len(ext_sents)) ext_sents.append(label) masks.append(label_mask) else: if idx[-1].item() != tmp_stop: ext_sents.append(t) masks.append(m) #ext_arts += [raw_art_sents[i] for i in ext] #print(ext_sents) #print(masks) #print(dirty) #exit(0) if beam_size > 1: #print(ext_sents) #print(masks) all_beams = abstractor(ext_sents, masks, beam_size, diverse) print('rerank') dec_outs = rerank_mp(all_beams, ext_inds) for d in dirty: dec_outs[d] = [] # TODO:!!!!!!!!!!! else: dec_outs = abstractor(ext_sents, masks) for d in dirty: dec_outs[d] = [] assert i == batch_size*i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j+n]] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) if i % 100 == 0: print(total_leng / total_num) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i/n_data*100, timedelta(seconds=int(time()-start)) ), end='') print()
def decode(save_path, abs_dir, ext_dir, split, batch_size, max_len, cuda): start = time() # setup model if abs_dir is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization abstractor = identity else: abstractor = Abstractor(abs_dir, max_len, cuda) if ext_dir is None: # NOTE: if no abstractor is provided then # it would be the lead-N extractor extractor = lambda art_sents: list(range(len(art_sents)))[:MAX_ABS_NUM] else: extractor = Extractor(ext_dir, max_ext=MAX_ABS_NUM, cuda=cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll ) # prepare save paths and logs for i in range(MAX_ABS_NUM): os.makedirs(join(save_path, 'output_{}'.format(i))) dec_log = {} dec_log['abstractor'] = (None if abs_dir is None else json.load(open(join(abs_dir, 'meta.json')))) dec_log['extractor'] = (None if ext_dir is None else json.load(open(join(ext_dir, 'meta.json')))) dec_log['rl'] = False dec_log['split'] = split dec_log['beam'] = 1 # greedy decoding only with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents) ext_inds += [(len(ext_arts), len(ext))] ext_arts += list(map(lambda i: raw_art_sents[i], ext)) dec_outs = abstractor(ext_arts) assert i == batch_size*i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j+n]] for k, dec_str in enumerate(decoded_sents): with open(join(save_path, 'output_{}/{}.dec'.format(k, i)), 'w') as f: f.write(make_html_safe(dec_str)) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i/n_data*100, timedelta(seconds=int(time()-start)) ), end='') print()
def decode(save_path, model_dir, split, batch_size, beam_size, diverse, max_len, cuda): start = time() if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) articles = [" ".join(article) for article in articles] return articles dataset = DecodeDataset(args.data_path, split) n_data = len(dataset) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll) # prepare save paths and logs os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['rl'] = True dec_log['split'] = split dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = tokenize(1000, raw_article_batch) batch_size = len(tokenized_article_batch) ext_inds = [] for num in range(batch_size): ext_inds += [(num, 1)] if beam_size > 1: all_beams = abstractor(tokenized_article_batch, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) else: dec_outs = abstractor(tokenized_article_batch) assert i == batch_size * i_debug for index in range(batch_size): decoded_sents = [ ' '.join(dec.split(",")) for dec in dec_outs[index] ] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe(' '.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') print()
def decode(args): save_path = args.path model_dir = args.model_dir batch_size = args.batch beam_size = args.beam diverse = args.div max_len = args.max_dec_word cuda = args.cuda start = time() # setup model with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = identity else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda) extractor = RLExtractor(model_dir, cuda=cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(args) n_data = len(dataset) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll) # prepare save paths and logs if not os.path.exists(join(save_path, 'output')): os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['abstractor'] = meta['net_args']['abstractor'] dec_log['extractor'] = meta['net_args']['extractor'] dec_log['rl'] = True dec_log['split'] = args.mode dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents)[:-1] # exclude EOE if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] else: ext = [i.item() for i in ext] ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] if beam_size > 1: all_beams = abstractor(ext_arts, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) else: dec_outs = abstractor(ext_arts) assert i == batch_size * i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j + n]] decoded_sents = decoded_sents[:20] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') print()
def decode(save_path, save_file, model_dir, split, batch_size, beam_size, diverse, max_len, cuda): start = time() # setup model with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = identity else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda) extractor = RLExtractor(model_dir, cuda=cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll) # prepare save paths and logs os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['abstractor'] = meta['net_args']['abstractor'] dec_log['extractor'] = meta['net_args']['extractor'] dec_log['rl'] = True dec_log['split'] = split dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents)[:-1] # exclude EOE if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] else: ext = [i.item() for i in ext] ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] if beam_size > 1: all_beams = abstractor(ext_arts, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) else: dec_outs = abstractor(ext_arts) assert i == batch_size * i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j + n]] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') print() #not needed for cnn/dailymail dataset probably f = open(save_file, "w") summaries_files = os.listdir(join(save_path, 'output')) n = len(summaries_files) summaries_list = [""] * n for fname in summaries_files: num = int(fname.replace(".dec", "")) f_local = open(join(save_path, "output", fname)) summaries_list[num] = f_local.read().replace("\n", " ") f_local.close() assert (len(summaries_list) == n) f.write("\n".join(summaries_list)) f.close()
def test(args, split): ext_dir = args.path ckpts = sort_ckpt(ext_dir) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader( dataset, batch_size=args.batch, shuffle=False, num_workers=4, collate_fn=coll ) # decode and evaluate top 5 models os.mkdir(join(args.path, 'decode')) os.mkdir(join(args.path, 'ROUGE')) for i in range(min(5, len(ckpts))): print('Start loading checkpoint {} !'.format(ckpts[i])) cur_ckpt = torch.load( join(ext_dir, 'ckpt/{}'.format(ckpts[i])) )['state_dict'] extractor = Extractor(ext_dir, cur_ckpt, args.emb_type, cuda=args.cuda) save_path = join(args.path, 'decode/{}'.format(ckpts[i])) os.mkdir(save_path) # decoding ext_list = [] cur_idx = 0 start = time() with torch.no_grad(): for raw_article_batch in loader: tokenized_article_batch = map(tokenize(None, args.emb_type), raw_article_batch) for raw_art_sents in tokenized_article_batch: ext_idx = extractor(raw_art_sents) ext_list.append(ext_idx) cur_idx += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( cur_idx, n_data, cur_idx/n_data*100, timedelta(seconds=int(time()-start)) ), end='') print() # write files for file_idx, ext_ids in enumerate(ext_list): dec = [] data_path = join(DATA_DIR, '{}/{}.json'.format(split, file_idx)) with open(data_path) as f: data = json.loads(f.read()) n_ext = 2 if data['source'] == 'CNN' else 3 n_ext = min(n_ext, len(data['article'])) for j in range(n_ext): sent_idx = ext_ids[j] dec.append(data['article'][sent_idx]) with open(join(save_path, '{}.dec'.format(file_idx)), 'w') as f: for sent in dec: print(sent, file=f) # evaluate current model print('Starting evaluating ROUGE !') dec_path = save_path ref_path = join(DATA_DIR, 'refs/{}'.format(split)) ROUGE = eval_rouge(dec_path, ref_path) print(ROUGE) with open(join(args.path, 'ROUGE/{}.txt'.format(ckpts[i])), 'w') as f: print(ROUGE, file=f)
def decode_entity(save_path, model_dir, split, batch_size, beam_size, diverse, max_len, cuda, sc, min_len): start = time() # setup model with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: #if not meta['net_args'].__contains__('abstractor'): # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = identity else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda, min_len=min_len) if sc: extractor = SCExtractor(model_dir, cuda=cuda, entity=True) else: extractor = RLExtractor(model_dir, cuda=cuda) # setup loader def coll(batch): batch = list(filter(bool, batch)) return batch if args.key == 1: key = 'filtered_rule1_input_mention_cluster' elif args.key == 2: key = 'filtered_rule23_6_input_mention_cluster' else: raise Exception dataset = DecodeDatasetEntity(split, key) n_data = len(dataset) loader = DataLoader( dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll ) # prepare save paths and logs os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['abstractor'] = meta['net_args']['abstractor'] dec_log['extractor'] = meta['net_args']['extractor'] dec_log['rl'] = True dec_log['split'] = split dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding if sc: i = 0 length = 0 sent_selected = 0 with torch.no_grad(): for i_debug, raw_input_batch in enumerate(loader): raw_article_batch, clusters = zip(*raw_input_batch) tokenized_article_batch = map(tokenize(None), raw_article_batch) #processed_clusters = map(preproc(list(tokenized_article_batch), clusters)) #processed_clusters = list(zip(*processed_clusters)) ext_arts = [] ext_inds = [] pre_abs = [] beam_inds = [] for raw_art_sents, raw_cls in zip(tokenized_article_batch, clusters): processed_clusters = preproc(raw_art_sents, raw_cls) ext = extractor((raw_art_sents, processed_clusters))[:] # exclude EOE sent_selected += len(ext) if not ext: # use top-3 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(3))[:len(raw_art_sents)] else: ext = [i for i in ext] ext_art = list(map(lambda i: raw_art_sents[i], ext)) pre_abs.append([word for sent in ext_art for word in sent]) beam_inds += [(len(beam_inds), 1)] if beam_size > 1: # all_beams = abstractor(ext_arts, beam_size, diverse) # dec_outs = rerank_mp(all_beams, ext_inds) all_beams = abstractor(pre_abs, beam_size, diverse=1.0) dec_outs = rerank_mp(all_beams, beam_inds) else: dec_outs = abstractor(pre_abs) for dec_out in dec_outs: dec_out = sent_tokenize(' '.join(dec_out)) ext = [sent.split(' ') for sent in dec_out] ext_inds += [(len(ext_arts), len(ext))] ext_arts += ext dec_outs = ext_arts assert i == batch_size*i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j+n]] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i/n_data*100, timedelta(seconds=int(time()-start)) ), end='') length += len(decoded_sents) else: i = 0 length = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents)[:-1] # exclude EOE if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] else: ext = [i.item() for i in ext] ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] if beam_size > 1: all_beams = abstractor(ext_arts, beam_size, diverse) dec_outs = rerank_mp(all_beams, ext_inds) else: dec_outs = abstractor(ext_arts) assert i == batch_size*i_debug for j, n in ext_inds: decoded_sents = [' '.join(dec) for dec in dec_outs[j:j+n]] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i/n_data*100, timedelta(seconds=int(time()-start)) ), end='') length += len(decoded_sents) print('average summary length:', length / i) print('average sentence selected:', sent_selected)
def decode(save_path, abs_dir, ext_dir, split, batch_size, max_len, cuda, min_len): start = time() # setup model if abs_dir is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization abstractor = identity else: #abstractor = Abstractor(abs_dir, max_len, cuda) abstractor = BeamAbstractor(abs_dir, max_len, cuda, min_len, reverse=args.reverse) if ext_dir is None: # NOTE: if no exstractor is provided then # it would be the lead-N extractor extractor = lambda art_sents: list(range(len(art_sents)))[:MAX_ABS_NUM] else: if args.no_force_ext: extractor = Extractor(ext_dir, max_ext=MAX_ABS_NUM, cuda=cuda, force_ext=not args.no_force_ext) else: extractor = Extractor(ext_dir, max_ext=MAX_ABS_NUM, cuda=cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll) os.makedirs(save_path) # prepare save paths and logs dec_log = {} dec_log['abstractor'] = (None if abs_dir is None else json.load( open(join(abs_dir, 'meta.json')))) dec_log['extractor'] = (None if ext_dir is None else json.load( open(join(ext_dir, 'meta.json')))) dec_log['rl'] = False dec_log['split'] = split if abs_dir is not None: dec_log['beam'] = 5 # greedy decoding only beam_size = 5 else: dec_log['beam'] = 1 beam_size = 1 with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) print(dec_log['extractor']) if dec_log['extractor']['net_args'][ 'stop'] == False and not args.no_force_ext: for i in range(MAX_ABS_NUM + 1): os.makedirs(join(save_path, 'output_{}'.format(i))) else: os.makedirs(join(save_path, 'output')) # Decoding i = 0 length = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] pre_abs = [] beam_inds = [] for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents) ext_art = list(map(lambda i: raw_art_sents[i], ext)) pre_abs.append([word for sent in ext_art for word in sent]) beam_inds += [(len(beam_inds), 1)] if beam_size > 1: all_beams = abstractor(pre_abs, beam_size, diverse=1.0) dec_outs = rerank_mp(all_beams, beam_inds) else: dec_outs = abstractor(pre_abs) for dec_out in dec_outs: dec_out = sent_tokenize(' '.join(dec_out)) ext = [sent.split(' ') for sent in dec_out] ext_inds += [(len(ext_arts), len(ext))] ext_arts += ext if dec_log['extractor']['net_args'][ 'stop'] == False and not args.no_force_ext: dec_outs = ext_arts assert i == batch_size * i_debug for j, n in ext_inds: decoded_sents = [ ' '.join(dec) for dec in dec_outs[j:j + n] ] for k, dec_str in enumerate(decoded_sents): if k > MAX_ABS_NUM - 2: break with open( join(save_path, 'output_{}/{}.dec'.format(k, i)), 'w') as f: f.write(make_html_safe(dec_str)) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') else: dec_outs = ext_arts assert i == batch_size * i_debug for j, n in ext_inds: decoded_sents = [ ' '.join(dec) for dec in dec_outs[j:j + n] ] with open(join(save_path, 'output/{}.dec'.format(i)), 'w') as f: f.write(make_html_safe('\n'.join(decoded_sents))) i += 1 print('{}/{} ({:.2f}%) decoded in {} seconds\r'.format( i, n_data, i / n_data * 100, timedelta(seconds=int(time() - start))), end='') length += len(decoded_sents) print('average summary length:', length / i)
def decode(save_path, model_dir, split, batch_size, beam_size, diverse, max_len, cuda): start = time() # setup model with open(join(model_dir, 'meta.json')) as f: meta = json.loads(f.read()) if meta['net_args']['abstractor'] is None: # NOTE: if no abstractor is provided then # the whole model would be extractive summarization assert beam_size == 1 abstractor = identity else: if beam_size == 1: abstractor = Abstractor(join(model_dir, 'abstractor'), max_len, cuda) else: abstractor = BeamAbstractor(join(model_dir, 'abstractor'), max_len, cuda) extractor = RLExtractor(model_dir, cuda=cuda) # setup loader def coll(batch): articles = list(filter(bool, batch)) return articles dataset = DecodeDataset(split) n_data = len(dataset) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=coll) # prepare save paths and logs # os.makedirs(join(save_path, 'output')) dec_log = {} dec_log['abstractor'] = meta['net_args']['abstractor'] dec_log['extractor'] = meta['net_args']['extractor'] dec_log['rl'] = True dec_log['split'] = split dec_log['beam'] = beam_size dec_log['diverse'] = diverse with open(join(save_path, 'log.json'), 'w') as f: json.dump(dec_log, f, indent=4) # Decoding i = 0 count = 0 with torch.no_grad(): for i_debug, raw_article_batch in enumerate(loader): tokenized_article_batch = map(tokenize(None), raw_article_batch) ext_arts = [] ext_inds = [] for raw_art_sents in tokenized_article_batch: ext = extractor(raw_art_sents)[:-1] # exclude EOE if not ext: # use top-5 if nothing is extracted # in some rare cases rnn-ext does not extract at all ext = list(range(5))[:len(raw_art_sents)] else: ext = [i.item() for i in ext] ext_inds += [(len(ext_arts), len(ext))] ext_arts += [raw_art_sents[i] for i in ext] if beam_size > 1: all_beams = abstractor(ext_arts, beam_size, diverse) for ind_file, (start, finish) in enumerate(ext_inds): article_beams = all_beams[start:start + finish] file = {} for ind_sent, sent in enumerate(article_beams): file[ind_sent] = defaultdict(list) sentence = " ".join(ext_arts[start + ind_sent]) file[ind_sent]['sentence'].append(sentence) for hypothesis in sent: file[ind_sent]['summarizer_logprob'].append( hypothesis.logprob) file[ind_sent]['hypotheses'].append(" ".join( hypothesis.sequence)) with open( os.path.join('exported_beams', '{}.json'.format(count + ind_file)), 'w') as f: json.dump(file, f, ensure_ascii=False) count += batch_size