def send_relevant_entry_updates(self,max_entries=4, decay=.9, context_utts=9, extract_top_n_keywords=10, min_found_keywords=3, min_transcript_utts=2):
print 'send_relevant_entry_updates called'
with Timer() as t:
#Do the decay for the displayed entries:
#TODO: handle duplicate keywords and updated scores
for entry in self.displayed_entries:
entry["score"] *= decay
# keywords = self.ke.getKeywordsDruid(self.complete_transcript[-1])
# Take last 10 utterances and combine them
most_recent_transcript = " ".join(self.complete_transcript[-context_utts:])
# Extract top 9 keywords
keywords = self.ke.extract_best_keywords(most_recent_transcript, n_words=extract_top_n_keywords)
print keywords
#abort if we found very little keywords and haven't seen enough utterances
if len(keywords) < min_found_keywords or len(self.complete_transcript) < min_transcript_utts:
return
# Extract top wiki articles
new_relevant_entries = wiki_search_es.extract_best_articles(keywords, n=max_entries)
print "-> Extracted top ", len(new_relevant_entries), " documents", [(entry["title"], entry["score"]) for entry in new_relevant_entries]
new_relevant_entries = dict(zip([entry["title"] for entry in new_relevant_entries],
[entry for entry in new_relevant_entries ] ) )
new_relevant_entries_set = set(new_relevant_entries)
relevant_entries_set = set(self.relevant_entries)
num_added = 0
#generate add relevant entries
for key in new_relevant_entries_set - relevant_entries_set:
entry = new_relevant_entries[key]
if self.addDisplayEntry("wiki", entry):
num_added += 1
for category in entry["categories"]:
self.categories[category] += 1
#now look for changed scores (happens if a keyword got more important and gets mentioned again)
for key in (new_relevant_entries_set & relevant_entries_set):
entry = new_relevant_entries[key]
if entry["score"] > self.relevant_entries[key]["score"]:
print "score change for:",entry["title"], self.relevant_entries[key]["score"], "->", entry["score"]
found_displayed_entry = False
for display_entry in self.displayed_entries:
#already displayed, we could delete and read it, to reflect the new placement
if display_entry["title"] == key:
found_displayed_entry = True
#self.delDisplayEntry("wiki", entry["title"])
#self.addDisplayEntry("wiki", entry)
break
if not found_displayed_entry:
#not displayed, try to see if the higher score gets results in a document that is more important
self.addDisplayEntry("wiki", entry)
for key in new_relevant_entries_set - relevant_entries_set:
self.relevant_entries[key] = new_relevant_entries[key]
topCategories_Event = self.topCategories()
print topCategories_Event
# TODO: only send something if topCategories actually changes
self.keyword_client.sendCategories(topCategories_Event)
print 'send_relevant_entry_updates finished. Time needed:', t.secs, 'seconds.'
print 'Displayed entries should now be:',[entry['title'] for entry in self.displayed_entries]
print 'Added:',num_added
# minimum_should_match_percent=5
print "extract_best_keywords:"
#extracted_num_tokens_like_manual = ke.extract_best_keywords(in_text, n_words=10, tfidf_only=tfidf_only) #was n_words=num_tokens
#print extracted_num_tokens_like_manual
extracted_10_tokens = ke.extract_best_keywords(in_text, n_words=10, tfidf_only=tfidf_only)
#if multiwords:
# #boost scores
# extracted_10_tokens = [(elem,score) for (elem,score) in extracted_10_tokens]
print extracted_10_tokens
# Extract top wiki articles
new_relevant_entries = wiki_search_es.extract_best_articles(extracted_10_tokens, n=10, min_summary_chars=300, minimum_should_match_percent=minimum_should_match_percent)
print "-> Extracted top ", len(new_relevant_entries), " documents", [(entry["title"], entry["score"]) for entry in new_relevant_entries]
# Write extracted tokens into file
with io.open(os.path.join(keyword_eval_dir, myfile), 'w', encoding='utf-8') as out_file:
out_file.write(u'\n'.join([' '.join(elem[0].split('_')) for elem in extracted_10_tokens])+u'\n')
with io.open(os.path.join(query_eval_dir, myfile), 'w', encoding='utf-8') as out_file:
out_file.write(wiki_search_es.construct_query_string(extracted_10_tokens)+u'\n')
json_out = {'filename':myfile, 'orig':orig, 'top10':new_relevant_entries}
print json_out
with open(os.path.join(ndcg_eval_dir, myfile[:-4]+'.json'), 'w') as outfile:
json.dump(json_out, outfile, indent=4)
start_time = time.time()
print 'Extracting keyphrases...'
tokens = ke.habibi_mimic(raw, n=num_tokens)
print 'Done extracting keyphrases. Time needed:', time.time() - start_time
print '==========Text=========='
for sentence in nltk.sent_tokenize(raw):
print sentence
print '==========Keyphrases=========='
print tokens
start_time = time.time()
print 'Retrieving best articles...'
results = wiki_search_es.extract_best_articles(tokens, n=10)
print 'Done retrieving articles. Time needed:', time.time() - start_time
# Convert resulting dictionary into sorted list
sorted_results = []
for key, value in results.iteritems():
sorted_results.append(value)
sorted_results = sorted(sorted_results, key=lambda item: item['score'], reverse=True)
for article in sorted_results:
print '==========Article=========='
print 'Title:', article['title']
#print 'Summary:', article['text']
#print 'URL:', article['url']
print 'Categories:', article['categories']
print 'Score:', article['score']
def send_relevant_entry_updates(self,max_entries=4, decay=.8):
print 'send_relevant_entry_updates called'
with Timer() as t:
#Do the decay for the displayed entries:
#TODO: handle duplicate keywords and updated scores
for entry in self.displayed_entries:
entry["score"] *= decay
# keywords = self.ke.getKeywordsDruid(self.complete_transcript[-1])
# Take last 10 utterances and combine them
most_recent_transcript = " ".join(self.complete_transcript[-10:])
# Extract top 9 keywords
keywords = self.ke.extract_best_keywords(most_recent_transcript, n=9)
print keywords
# Extract top 5 wiki articles
new_relevant_entries = wiki_search_es.extract_best_articles(keywords, n=5)
new_relevant_entries_set = set(new_relevant_entries)
relevant_entries_set = set(self.relevant_entries)
#generate del relevant entries
#for key in relevant_entries_set - new_relevant_entries_set:
# entry = self.relevant_entries[key]
# self.delDisplayEntry("wiki", entry["title"])
#generate add relevant entries
for key in new_relevant_entries_set - relevant_entries_set:
entry = new_relevant_entries[key]
if self.addDisplayEntry("wiki", entry):
for category in entry["categories"]:
self.categories[category] += 1
#now look for changed scores (happens if a keyword got more important and gets mentioned again)
for key in (new_relevant_entries_set & relevant_entries_set):
entry = new_relevant_entries[key]
if entry["score"] > self.relevant_entries[key]["score"]:
print "score change for:",entry["title"], self.relevant_entries[key]["score"], "->", entry["score"]
found_displayed_entry = False
for display_entry in self.displayed_entries:
#already displayed, we could delete and read it, to reflect the new placement
if display_entry["title"] == key:
found_displayed_entry = True
#self.delDisplayEntry("wiki", entry["title"])
#self.addDisplayEntry("wiki", entry)
break
if not found_displayed_entry:
#not displayed, try to see if the higher score gets results in a document that is more important
self.addDisplayEntry("wiki", entry)
for key in new_relevant_entries_set - relevant_entries_set:
self.relevant_entries[key] = new_relevant_entries[key]
topCategories_Event = self.topCategories()
print topCategories_Event
# TODO: only send something if topCategories actually changes
self.keyword_client.sendCategories(topCategories_Event)
print 'send_relevant_entry_updates finished. Time needed:', t.secs, 'seconds.'
print 'Displayed entries should now be:',[entry['title'] for entry in self.displayed_entries]