def readDatasets(datasets, samples=9999):
### read documents and ref. summaries
summaries = []
targets = []
groups = []
sentences_of_topics = {}
sorted_idxs_list = []
models_list = []
docs_list = []
topic_list = []
for dataset in datasets:
reader = CorpusReader(PROCESSED_PATH)
data = reader.get_data(dataset)
sample_num = samples
ml = []
dl = []
tl = []
### read data
for topic, docs, models in data:
print('read DATA {}, TOPIC {}'.format(dataset, topic))
summs, ref_values_dic = readSummaries(dataset, topic,
'ground_truth', sample_num)
ref_rewards = aggregateScores(ref_values_dic)
summs = summs[:len(ref_rewards)]
print('num of summaries read: {}'.format(len(summs)))
sentences = [
sent2tokens(sentence, 'english') for _, doc in docs
for sentence in doc
]
sentences_of_topics[topic] = sentences
ml.append(models)
dl.append(docs)
tl.append(topic)
sorted_idxs = np.argsort(np.array(ref_rewards))
sorted_idxs_list.extend(sorted_idxs)
summaries.extend(summs)
targets.extend(ref_rewards)
groups.extend(['{}-{}'.format(dataset, topic)] * len(summs))
models_list.append(ml)
docs_list.append(dl)
topic_list.append(tl)
summaries = np.array(summaries)
targets = np.array(targets)
groups = np.array(groups)
return summaries, targets, groups, models_list, docs_list, topic_list, sentences_of_topics, sorted_idxs_list
def build_duc_vocabulary():
datasets = ['DUC2001', 'DUC2002', 'DUC2004']
sample_num = 9999
cv_fold_num = 10
validation_size = 0.1
### read documents and ref. summaries
reader = CorpusReader(PROCESSED_PATH)
vocabulary = set()
for dataset in datasets:
data = reader.get_data(dataset)
### read data
for topic, docs, models in data:
print('read DATA {}, TOPIC {}'.format(dataset, topic))
summs, ref_values_dic = readSummaries(dataset, topic, 'rouge',
sample_num)
sentences = [
sent2tokens(sentence, 'english') for _, doc in docs
for sentence in doc
]
vocabulary.update(
[token for sentence in sentences for token in sentence])
return vocabulary
def writeNER(dataset):
summary_len = 100
reader = CorpusReader(PROCESSED_PATH)
data = reader.get_data(dataset, summary_len)
nlp = StanfordCoreNLP(CORENLP_PATH)
base_path = os.path.join(PROCESSED_PATH, dataset)
topic_cnt = 0
for topic, docs, models in data:
topic_cnt += 1
topic_path = os.path.join(base_path, topic, 'docs.ner')
if not os.path.exists(topic_path):
os.mkdir(topic_path)
for dd in docs:
dname = dd[0].split('/')[-1].strip()
print('{} topic {}, doc {}'.format(topic_cnt, topic, dname))
output = ''
for sen in dd[1]:
ner = nlp.ner(sen)
output += repr(ner) + '\n'
write_to_file(output, os.path.join(topic_path, dname))
nlp.close()
#print('sentences: {}\n{}'.format(self.sentences[sum_idx[i]], self.sentences[sum_idx[j]]))
#print('vec1 : {}\n vec2: {}'.format(self.sent_vecs[sum_idx[i]], self.sent_vecs[sum_idx[j]]))
red_scores[-1] = 0.
return np.mean(red_scores)
if __name__ == '__main__':
dataset = 'DUC2002' ## DUC2001, DUC2002, DUC2004
sample_num = 9999
out_base = os.path.join(FEATURE_DIR, dataset)
if not os.path.exists(out_base):
os.makedirs(out_base)
### read documents and ref. summaries
reader = CorpusReader(PROCESSED_PATH)
data = reader.get_data(dataset)
### store all results
all_test_reward_dic = OrderedDict()
topic_cnt = 0
summaries = []
targets = []
groups = []
models_list = []
docs_list = []
### read data
infersent = InfersentRewardGenerator()
for topic, docs, models in data:
if len(sys.argv) == 4:
dataset = sys.argv[1]
start = int(sys.argv[2])
end = int(sys.argv[3])
else:
dataset = 'DUC2001' #DUC2001, DUC2002, DUC2004
start = 0
end = 10000
language = 'english'
summary_len = 100
summary_num = 10001
base_dir = os.path.join(SUMMARY_DB_DIR, dataset)
reader = CorpusReader(PROCESSED_PATH)
data = reader.get_data(dataset, summary_len)
topic_cnt = 0
for topic, docs, models in data:
topic_cnt += 1
if not (topic_cnt > start and topic_cnt <= end):
continue
dir_path = os.path.join(base_dir, topic)
if not os.path.exists(dir_path):
os.makedirs(dir_path)
vec = Vectoriser(docs, summary_len)
print('-----Generate samples for topic {}: {}-----'.format(
topic_cnt, topic))
def correlation(features, sizes):
names = []
for f, s in zip(features, sizes):
if s > 1:
for i in range(s):
names.append(f + str(i))
else:
names.append(f)
names.append('rouge_reward')
dataset = 'DUC2001' ## DUC2001, DUC2002, DUC2004
sample_num = 9999
bin_num = 20
cv_fold_num = 10
### read documents and ref. summaries
reader = CorpusReader(PROCESSED_PATH)
data = reader.get_data(dataset)
topic_cnt = 0
summaries = []
groups = []
models_list = []
docs_list = []
targets = []
### read data
for topic, docs, models in data:
print('read DATA {}, TOPIC {}'.format(dataset, topic))
summs, ref_values_dic = readSummaries(dataset, topic, 'rouge',
sample_num)
print('num of summaries read: {}'.format(len(summaries)))
ref_rewards = aggregateScores(ref_values_dic)
models_list.append(models)
docs_list.append(docs)
summaries.extend(summs)
groups.extend([topic] * len(summs))
targets.extend(ref_rewards)
topic_cnt += 1
allFeatures = readFeatures(features, dataset, np.array(summaries), groups,
set(groups))
allFeatures = np.c_[allFeatures, np.array(targets)]
correlations = {}
threshold_correlation = {}
for col1, col2 in itertools.combinations(range(len(names)), 2):
pcc = pearsonr(allFeatures[:, col1], allFeatures[:, col2])[0]
correlations[names[col1] + ' ' + names[col2] + ': pcc = '] = pcc
# other way for ease of reading
correlations[names[col2] + ' ' + names[col1] + ': pcc = '] = pcc
if pcc < -0.8:
threshold_correlation[names[col1] + ' ' + names[col2] +
': pcc = '] = pcc
threshold_correlation[names[col2] + ' ' + names[col1] +
': pcc = '] = pcc
#for key in sorted(correlations.keys()):
# print(key+str(correlations[key]))
print("Pairs with pcc >.9")
for key in sorted(threshold_correlation.keys()):
print(key + str(threshold_correlation[key]))