def more_like_this2(self, item_doc, result_num):
similar_questions = []
if not item_doc:
item_doc.append(ResultItem(None, 1.0, "No Title", 0))
query = ""
if item_doc.doc:
query += self.document_to_query(item_doc.doc)
query = remove_unified_stop_lists(query)
queryparser = QueryParser(Version.LUCENE_CURRENT, "term",
self.analyzer)
if query:
try:
like_query = queryparser.parse(query)
hits = self.searcher.search(like_query, result_num).scoreDocs
for i, hit in enumerate(hits):
doc = self.searcher.doc(hit.doc)
similar_questions.append(doc.get("question_id"))
except Exception as e:
print "Question Searcher: Error: %s" % e
# write_search_log("Question Searcher: Error: %s" % e + "\n")
print(traceback.format_exc())
# self.searchermgr.decRef(self.searcher)
# self.searchermgr.release(self.searcher)
# self.searcher = None
# self.directory.close()
# self.directory = None
return similar_questions
# def release(self, searcher):
def Generator(code):
file_content = code
print '1. Origianl Query : ', file_content
ast = parse(file_content, resolve=False)
query = add_code_keyword_into_document(file_content, ast)
print "2. Right after alternation & before the removing stop words : ", query
query = remove_unified_stop_lists(query)
print '3. Right after the stop words removing : ', query
return query
def more_like_this2(self, limit, item_doc, score_logs_for_each, user_query,
flag): #flag = UQ(1) or not(0)
bench_result = []
query = ""
if flag == 1:
query += user_query
# item_doc = ResultItem(None, 0.0, "No Title", 'None','None', None)
if flag == 0 and item_doc.doc:
query += self.document_to_query(item_doc.doc)
query = remove_unified_stop_lists(query)
queryparser = QueryParser(Version.LUCENE_CURRENT, "typed_method_call",
self.analyzer)
if query:
try:
parsed_query = queryparser.parse(query)
hits = self.searcher.search(parsed_query, limit).scoreDocs
temp = 1
for i, hit in enumerate(hits):
doc = self.searcher.doc(hit.doc)
matched = doc.get('file').split('/')[9].split('.')[0]
score_logs_for_each += str(matched) + '\t' + str(
round(hit.score, 2)) + '\n'
matched_terms = self.get_matched_keywords2(
parsed_query, hit.doc)
# print "Matched Terms : ", matched_terms
# print("File %s" % temp, doc.get("file"), "//", doc.get("file_content"))
temp += 1
file_path = doc.get("file")
content = None
try:
with open(file_path) as f:
content = f.read()
except:
pass
if content:
item = BenchResultItem(doc.get("file"), content,
matched_terms,
hit.score, item_doc,
doc.get("line_numbers"),
hit.doc)
bench_result.append(item)
except Exception as e:
print "BenchSearcher Error: %s" % e
print(traceback.format_exc())
# self.searchermgr.release()
# self.searcher = None
# self.directory.close()
# self.directory = None
return bench_result, score_logs_for_each
def more_like_this3(self, limit, score_logs_for_each, user_query):
query = ""
bench_result = []
# if not item_doc:
# item_doc.append(ResultItem(None, 1.0, "No Title", 0, 0))
# if item_doc.doc:
# query += self.document_to_query(item_doc.doc)
query += user_query
query = remove_unified_stop_lists(query)
queryparser = QueryParser(Version.LUCENE_CURRENT, "typed_method_call",
self.analyzer)
if query:
try:
parsed_query = queryparser.parse(query)
hits = self.searcher.search(parsed_query, limit).scoreDocs
temp = 1
for i, hit in enumerate(hits):
score_logs_for_each += str(round(hit.score, 2)) + '\n'
doc = self.searcher.doc(hit.doc)
matched_terms = self.get_matched_keywords2(
parsed_query, hit.doc)
# print "Matched Terms : ", matched_terms
# print("File %s" % temp, doc.get("file"), "//", doc.get("file_content"))
temp += 1
file_path = doc.get("file")
content = None
try:
with open(file_path) as f:
content = f.read()
except:
pass
if content:
item = BenchResultItem_UQ(doc.get("file"), content,
matched_terms, hit.score,
doc.get("line_numbers"),
hit.doc)
bench_result.append(item)
except Exception as e:
print "BenchSearcher Error: %s" % e
print(traceback.format_exc())
# self.searchermgr.decRef(self.searcher)
# self.searchermgr.release(self.searcher)
# self.searcher = None
# self.directory.close()
# self.directory = None
return bench_result, score_logs_for_each
def Generator(code):
file_content = code
# print '1. Origianl Query : ', file_content
ast = parse(file_content, resolve=False) # newJavaParser를 사용하여 자바 코드 파싱
query = add_code_keyword_into_document(file_content, ast)
# print "Query before the removing stop words : ", query
# write_search_log("\nQuery before the removing stop words : " + str(query))
# print '2. Right after the code query generator : ', query
query = remove_unified_stop_lists(query)
# print '3. Right after the stop words removing : ', query
# print "Transformed user code query : ", query
# write_search_log("\nTransformed user code query : " + str(query))
return query
def more_like_this2(self, limit, item_doc, user_query,
flag): #flag = UQ(1) or not(0)
results = []
query = ""
if flag == 1:
query += user_query
# item_doc = ResultItem(None, 0.0, "No Title", 'None','None', None)
if flag == 0 and item_doc.doc:
query += self.document_to_query(item_doc.doc)
query = remove_unified_stop_lists(query)
queryparser = QueryParser(Version.LUCENE_CURRENT, "typed_method_call",
self.analyzer)
if query:
try:
parsed_query = queryparser.parse(query)
hits = self.searcher.search(parsed_query, limit).scoreDocs
temp = 1
for i, hit in enumerate(hits):
doc = self.searcher.doc(hit.doc)
matched_terms = self.get_matched_keywords2(
parsed_query, hit.doc)
temp += 1
file_path = doc.get("file")
content = None
try:
with open(file_path) as f:
content = f.read()
except:
pass
if content:
item = GithubResultItem(doc.get("file"), content,
matched_terms, hit.score,
item_doc,
doc.get("line_numbers"),
hit.doc)
results.append(item)
except Exception as e:
print "GitHub Searcher Error: %s" % e
print(traceback.format_exc())
return results
def more_like_this2(self, limit, score_logs_for_each, user_query, flag):
bench_result = []
query = ""
if flag == 1:
query += user_query
query = remove_unified_stop_lists(query)
queryparser = QueryParser(Version.LUCENE_CURRENT, "typed_method_call", self.analyzer)
if query:
try:
parsed_query = queryparser.parse(query)
hits = self.searcher.search(parsed_query, limit).scoreDocs
temp = 1
for i, hit in enumerate(hits):
doc = self.searcher.doc(hit.doc)
matched = doc.get('file').split('/')[9].split('.')[0]
score_logs_for_each += str(matched) + '\t' + str(round(hit.score, 2)) + '\n'
matched_terms = self.get_matched_keywords2(parsed_query, hit.doc)
temp += 1
file_path = doc.get("file")
content = None
try:
with open(file_path) as f:
content = f.read()
except:
pass
if content:
item = BenchResultItem_UQ(doc.get("file"), content, matched_terms, hit.score, doc.get("line_numbers"), hit.doc)
bench_result.append(item)
except Exception as e:
print "BenchSearcher Error: %s" % e
print(traceback.format_exc())
return bench_result, score_logs_for_each
def more_like_this2(
self, item_doc, result_num
): #들어온 질문 docs들에 대해 순회하면서 최종 query로 생성하고 Question Index에서 비슷한거 검색할 것.
similar_questions = []
if not item_doc:
item_doc.append(ResultItem(None, 1.0, "No Title", 0))
query = ""
if item_doc.doc:
query += self.document_to_query(item_doc.doc)
query = remove_unified_stop_lists(query)
queryparser = QueryParser(Version.LUCENE_CURRENT, "term",
self.analyzer)
if query: #########이 시점에서의 Unified Query는 Tokenization, Stemming 이 되어있음..########
try:
like_query = queryparser.parse(query)
hits = self.searcher.search(
like_query,
result_num).scoreDocs #Q와 비슷한 Q들 상위 3개씩의 결과 그럼 총 9개
for i, hit in enumerate(hits):
doc = self.searcher.doc(hit.doc)
similar_questions.append(doc.get("question_id"))
except Exception as e:
print "Question Searcher: Error: %s" % e
# write_search_log("Question Searcher: Error: %s" % e + "\n")
print(traceback.format_exc())
# self.searchermgr.decRef(self.searcher)
# self.searchermgr.release(self.searcher)
# self.searcher = None
# self.directory.close()
# self.directory = None
return similar_questions
def more_like_this2(self, limit, item_doc, user_query):
github_result = []
if not item_doc:
item_doc.append(ResultItem(None, 1.0, "No Title", 0, 0))
query = ""
if item_doc.doc:
query += self.document_to_query(item_doc.doc)
query += user_query
query = remove_unified_stop_lists(query)
print '................................................................................................'
print "Project Searcher Unified Query :", query
print '................................................................................................'
write_search_log(
"................................................................................................\n"
+ "Project Searcher Unified Query : " +
str(query.encode('utf-8')) + "\n" +
"................................................................................................\n"
)
queryparser = QueryParser(Version.LUCENE_CURRENT, "typed_method_call",
self.analyzer)
if query:
try:
like_query = queryparser.parse(query)
hits = self.searcher.search(like_query,
limit).scoreDocs #answer 1개당 10개씩
temp = 1
for i, hit in enumerate(hits):
doc = self.searcher.doc(hit.doc)
matched_terms = self.get_matched_keywords2(
like_query, hit.doc)
#print "Matched Terms : ", matched_terms
print("File %s" % temp, doc.get("file"), "//",
doc.get("file_content")
) #, "line_numbers", doc.get("line_numbers"))
write_search_log("File " + str(temp) +
str(doc.get("file")) + "//" +
str(doc.get("file_content")) + "\n")
temp += 1
file_path = doc.get("file")
print 'file_path = ', file_path
content = None
try:
with open(file_path) as f:
content = f.read()
except:
print "CAN'T OPEN THE FILE"
pass
if content:
item = GithubResultItem(doc.get("file"), content,
matched_terms, hit.score,
item_doc,
doc.get("line_numbers"),
hit.doc)
# print item.score
github_result.append(item)
except Exception as e:
print "GitSearcher Error: %s" % e
print(traceback.format_exc())
#sorted(github_result, key=attrgetter())
print 'github_result : ', github_result
return github_result