def buildIssuePart():
repos = linkOperator.selectRepoOver(5000)
textCorpus = open('issueText.dat', "w")
codeCorpus = open('issueCode.dat', "w")
try:
print 'start'
for highRepo in repos:
try:
issues = mysqlOperator.selectAllIssueInOneRepo(highRepo[0])
print highRepo[0], ":", len(issues)
for issue in issues:
titleWords = preprocessor.preprocessToWord(
issue[4].decode('utf-8'))
if len(titleWords):
# 不是空列表
for word in titleWords:
textCorpus.write(word.encode('utf-8'))
textCorpus.write(" ")
textCorpus.write("\n")
if issue[5]:
body = preprocessor.processHTML(
issue[5].decode('utf-8'))
bodyWords = body[1]
codeWords = body[0]
if len(bodyWords):
# 不是空列表
for word in bodyWords:
textCorpus.write(word.encode('utf-8'))
textCorpus.write(" ")
textCorpus.write("\n")
if len(codeWords):
# 不是空列表
for word in codeWords:
codeCorpus.write(word.encode('utf-8'))
codeCorpus.write(" ")
codeCorpus.write("\n")
comments = mysqlOperator.selectCommentInOneIssue(issue[1])
for comment in comments:
temp = preprocessor.processHTML(
comment[4].decode('utf-8'))
cBodyWords = temp[1]
cCodeWords = temp[0]
if len(cBodyWords):
# 不是空列表
for word in cBodyWords:
textCorpus.write(word.encode('utf-8'))
textCorpus.write(" ")
textCorpus.write("\n")
if len(cCodeWords):
# 不是空列表
for word in cCodeWords:
codeCorpus.write(word.encode('utf-8'))
codeCorpus.write(" ")
codeCorpus.write("\n")
except BaseException, e:
print "***", highRepo[0], ":", e
print traceback.format_exc()
print 'end'
def text2vec(text, isHtml):
if isHtml:
words = preprocessor.processHTML(text)[1]
else:
words = preprocessor.preprocessToWord(text)
res = []
for word in words:
try:
res.append(wordModel[word])
except KeyError:
res.append(np.zeros(VECTOR_SIZE))
return res
# -*- coding: UTF-8 -*-
from preprocessor import preprocessor
import re
print preprocessor.processHTML('''
Examples shown in the javadoc for TESD_DSAFSA_DDS <code>ReplayingDecoder.addOption</code> seems to be wrong. In the document it shows <code>IntegerHeaderFrameDecoder, MyDecoder</code> taking multiple parameters where as in reality it can only accept one. I'm working with versions 4.0.0.CR3, 4.0.0.CR5.
''')
print preprocessor.processDiffCode('''
@@ -349 +349 @@ public class JavadocUtilsTest {
- "HTML_COMMENT", JavadocUtils.getTokenName(20077));
+ "HTML_COMMENT", JavadocUtils.getTokenName(20078));
''')
print preprocessor.preprocessToWord('''
Examples shown in the javadoc for TESD_DSAFSA_DDS ReplayingDecoder.addOption seems to be wrong. In the document it shows IntegerHeaderFrameDecoder, MyDecoder taking multiple parameters where as in reality it can only accept one. I'm working with versions 4.0.0.CR3, 4.0.0.CR5.
''')
if None:
print 'none'
else:
print 'other'
# camelCase1 = re.compile(r'^[A-Z]+[a-z]+.*[A-Z]+.*$') # 3
# camelCase2 = re.compile(r'^[a-z]+.*[A-Z]+.*$') # 12
# upperCase = re.compile(r'^[A-Z]+[0-9]*$') # 7
# upperExtCase = re.compile(r'^[A-Z]*(_+[A-Z]*)+[0-9]*$') # 6
#
# print re.match(upperExtCase, 'aOption'), '1'
# print re.match(upperExtCase, 'addOption'), '2'
# print re.match(upperExtCase, 'AddToDeal'), '3'
def text2vec(text, isHtml):
if isHtml:
words = preprocessor.processHTML(text)[1]
else:
words = preprocessor.preprocessToWord(text)
return docModel.infer_vector(words)
def buildIssueAndCommit():
repos = linkOperator.selectOneRepo(50904245)
# repos = linkOperator.selectRepoOver(5000)
textCorpus = open('text50904245.dat', "w")
codeCorpus = open('code50904245.dat', "w")
try:
print 'start'
for highRepo in repos:
try:
# commit part
path = getPath(highRepo[1])
gitRe = gitResolver.GitResolver(path)
commits = gitRe.getCommits()
print path, ":", len(commits)
for commit in commits:
words = preprocessor.preprocessToWord(
commit.message.decode('utf-8'))
if len(words):
# 不是空列表
for word in words:
textCorpus.write(word.encode('utf-8'))
textCorpus.write(" ")
textCorpus.write("\n")
diffs = gitRe.getOneDiff(commit)
for diff in diffs:
diffCode = preprocessor.processDiffCode(diff.diff)
preDiffCode = preprocessor.processPreDiffCode(
diff.diff)
if len(diffCode):
for code in diffCode:
codeCorpus.write(code)
codeCorpus.write(" ")
codeCorpus.write("\n")
if len(preDiffCode):
for code in preDiffCode:
codeCorpus.write(code)
codeCorpus.write(" ")
codeCorpus.write("\n")
# issue part
issues = mysqlOperator.selectAllIssueInOneRepo(highRepo[0])
print highRepo[0], ":", len(issues)
for issue in issues:
titleWords = preprocessor.preprocessToWord(
issue[4].decode('utf-8'))
if len(titleWords):
# 不是空列表
for word in titleWords:
textCorpus.write(word.encode('utf-8'))
textCorpus.write(" ")
textCorpus.write("\n")
if issue[5]:
body = preprocessor.processHTML(
issue[5].decode('utf-8'))
bodyWords = body[1]
codeWords = body[0]
if len(bodyWords):
# 不是空列表
for word in bodyWords:
textCorpus.write(word.encode('utf-8'))
textCorpus.write(" ")
textCorpus.write("\n")
if len(codeWords):
# 不是空列表
for word in codeWords:
codeCorpus.write(word.encode('utf-8'))
codeCorpus.write(" ")
codeCorpus.write("\n")
comments = mysqlOperator.selectCommentInOneIssue(issue[1])
for comment in comments:
temp = preprocessor.processHTML(
comment[4].decode('utf-8'))
cBodyWords = temp[1]
cCodeWords = temp[0]
if len(cBodyWords):
# 不是空列表
for word in cBodyWords:
textCorpus.write(word.encode('utf-8'))
textCorpus.write(" ")
textCorpus.write("\n")
if len(cCodeWords):
# 不是空列表
for word in cCodeWords:
codeCorpus.write(word.encode('utf-8'))
codeCorpus.write(" ")
codeCorpus.write("\n")
except BaseException, e:
print "***", highRepo[0], ":", e
print traceback.format_exc()
print 'end'
diffCodeList.append((codeModel.infer_vector(diffCode), codeModel.infer_vector(preDiffCode)))
# code part init
codeMax = -1
tempMap['commitCode'] = None
tempMap['issueCode'] = None
# text part init
commitText = preprocessor.preprocessToWord(commit.message.decode('utf-8'))
commitTextVec = textModel.infer_vector(commitText)
tempMap['commitText'] = commitTextVec # 确定不变
titleWords = preprocessor.preprocessToWord(issue[4].decode('utf-8'))
tempMap['issueText'] = textModel.infer_vector(titleWords) # 可能改变
textMax = similarity(commitTextVec, tempMap['issueText'])
# issue body
if issue[5]:
body = preprocessor.processHTML(issue[5].decode('utf-8'))
bodyTextVec = textModel.infer_vector(body[1])
sim = similarity(commitTextVec, bodyTextVec)
if sim > textMax:
tempMap['issueText'] = bodyTextVec
textMax = sim
if len(body[0]) > 0:
codeVec = codeModel.infer_vector(body[0])
for diffCodeTemp in diffCodeList:
codeSim = similarity(codeVec, diffCodeTemp[0])
if codeSim > codeMax:
tempMap['commitCode'] = diffCodeTemp[0]
tempMap['issueCode'] = codeVec
codeMax = codeSim
preCodeSim = similarity(codeVec, diffCodeTemp[1])
if preCodeSim > codeMax: